215 lines
8.0 KiB
Python
215 lines
8.0 KiB
Python
"""Baut aus den GDELT-Feeds eine story-zentrierte JSON-Datei fuer die Website.
|
|
|
|
python build_site.py # heute (UTC)
|
|
python build_site.py --date 20260901 --limit 120
|
|
|
|
Joint die Event-Tabelle mit den Artikeltiteln aus dem GKG-Feed, verdichtet
|
|
auf Artikel-Ebene und uebersetzt die CAMEO-Codes in Klartext.
|
|
"""
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import io
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.request
|
|
import zipfile
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
from load_europe import EUROPE, TRANSCONTINENTAL, fetch_slot, slots
|
|
from tracker import NUMERIC
|
|
|
|
GKG_URL = "http://data.gdeltproject.org/gdeltv2/{}.gkg.csv.zip"
|
|
CAMEO_JSON = (Path(__file__).parent / '.venv' / 'lib' / 'python3.14'
|
|
/ 'site-packages' / 'data' / 'cameoCodes.json')
|
|
|
|
# QuadClass: verbal/material x kooperativ/konfliktiv
|
|
QUAD = {'1': 'verbal kooperativ', '2': 'material kooperativ',
|
|
'3': 'verbal konfliktiv', '4': 'material konfliktiv'}
|
|
|
|
TITLE_RE = re.compile(r'<PAGE_TITLE>(.*?)</PAGE_TITLE>', re.S)
|
|
|
|
|
|
def cameo_labels():
|
|
"""{EventCode: (Beschreibung, GoldsteinScale)} aus dem CAMEO-Codebook."""
|
|
raw = json.loads(CAMEO_JSON.read_text())
|
|
df = pd.DataFrame(raw)
|
|
return {str(k): (v['Description'], v['GoldsteinScale'])
|
|
for k, v in df.to_dict('index').items()}
|
|
|
|
|
|
def fetch_titles(stamp):
|
|
"""{url: titel} aus einem GKG-Paket. Leer, wenn das Paket fehlt."""
|
|
try:
|
|
with urllib.request.urlopen(GKG_URL.format(stamp), timeout=120) as r:
|
|
raw = r.read()
|
|
except urllib.error.HTTPError as err:
|
|
if err.code == 404:
|
|
return {}
|
|
raise
|
|
z = zipfile.ZipFile(io.BytesIO(raw))
|
|
text = z.open(z.namelist()[0]).read().decode('utf-8', 'replace')
|
|
out = {}
|
|
for line in text.split('\n'):
|
|
cols = line.split('\t')
|
|
if len(cols) < 27:
|
|
continue
|
|
m = TITLE_RE.search(cols[-1])
|
|
if m:
|
|
out[cols[4]] = html_unescape(m.group(1).strip())
|
|
return out
|
|
|
|
|
|
def html_unescape(s):
|
|
import html
|
|
# GKG kodiert doppelt: — und &#x2014; kommen beide vor
|
|
return html.unescape(html.unescape(s))
|
|
|
|
|
|
def build_stories(events, titles, cameo, limit):
|
|
"""Verdichtet Event-Zeilen auf Artikel-Ebene."""
|
|
events = events.copy()
|
|
events['title'] = events['SOURCEURL'].map(titles)
|
|
events['domain'] = events['SOURCEURL'].str.extract(r'https?://(?:www\.)?([^/]+)')
|
|
|
|
stories = []
|
|
for url, grp in events.groupby('SOURCEURL'):
|
|
top = grp.loc[grp['NumArticles'].idxmax()]
|
|
seen = set()
|
|
acts = []
|
|
for _, e in grp.sort_values('NumArticles', ascending=False).iterrows():
|
|
desc, gold = cameo.get(str(e.EventCode), (f'CAMEO {e.EventCode}', None))
|
|
key = (e.Actor1Name, e.Actor2Name, e.EventCode)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
acts.append({
|
|
'code': e.EventCode, 'label': desc,
|
|
'goldstein': None if pd.isna(e.GoldsteinScale) else float(e.GoldsteinScale),
|
|
'quad': QUAD.get(str(e.QuadClass), ''),
|
|
'actor1': None if pd.isna(e.Actor1Name) else e.Actor1Name.title(),
|
|
'actor2': None if pd.isna(e.Actor2Name) else e.Actor2Name.title(),
|
|
})
|
|
|
|
stories.append({
|
|
'url': url,
|
|
'title': top['title'] if isinstance(top['title'], str) else None,
|
|
'domain': top['domain'],
|
|
'country': top['ActionGeo_Country'],
|
|
'place': top['ActionGeo_FullName'],
|
|
'lat': None if pd.isna(top['ActionGeo_Lat']) else float(top['ActionGeo_Lat']),
|
|
'lon': None if pd.isna(top['ActionGeo_Long']) else float(top['ActionGeo_Long']),
|
|
'seen': str(top['DATEADDED']),
|
|
'articles': int(grp['NumArticles'].max()),
|
|
'mentions': int(grp['NumMentions'].max()),
|
|
'tone': round(float(grp['AvgTone'].mean()), 2),
|
|
'goldstein': round(float(grp['GoldsteinScale'].mean()), 2),
|
|
'events': acts,
|
|
'event_rows': len(grp),
|
|
})
|
|
|
|
stories = cluster_by_title(stories)
|
|
# Stories ohne Artikeltitel sind fuer Leser wertlos - der GKG-Join hat
|
|
# sie nicht gefunden, weil der Artikel aus einem aelteren Paket stammt.
|
|
stories = [s for s in stories if s['title']]
|
|
stories.sort(key=lambda s: (-s['articles'], -s['mentions']))
|
|
return stories[:limit]
|
|
|
|
|
|
def norm_title(t):
|
|
"""Titel auf Vergleichsform bringen: Kleinschreibung, ohne Satzzeichen
|
|
und ohne den bei vielen Quellen angehaengten Blattnamen."""
|
|
t = t.lower().split(' - ')[0].split(' | ')[0]
|
|
return re.sub(r'[^a-z0-9]+', ' ', t).strip()
|
|
|
|
|
|
def cluster_by_title(stories):
|
|
"""Syndizierte Agenturmeldungen zusammenfassen: gleicher Titel, andere Quelle.
|
|
|
|
Der Cluster behaelt die staerkste Variante; die uebrigen Quellen haengen
|
|
als 'sources' daran, ihre Events werden mit eingesammelt.
|
|
"""
|
|
clusters = {}
|
|
for s in stories:
|
|
key = norm_title(s['title']) if s['title'] else s['url']
|
|
clusters.setdefault(key, []).append(s)
|
|
|
|
merged = []
|
|
for group in clusters.values():
|
|
group.sort(key=lambda s: -s['articles'])
|
|
lead = dict(group[0])
|
|
lead['sources'] = [{'domain': g['domain'], 'url': g['url']} for g in group]
|
|
lead['event_rows'] = sum(g['event_rows'] for g in group)
|
|
if len(group) > 1:
|
|
seen = set()
|
|
acts = []
|
|
for g in group:
|
|
for e in g['events']:
|
|
key = (e['actor1'], e['actor2'], e['code'])
|
|
if key not in seen:
|
|
seen.add(key)
|
|
acts.append(e)
|
|
lead['events'] = acts
|
|
lead['tone'] = round(sum(g['tone'] for g in group) / len(group), 2)
|
|
merged.append(lead)
|
|
return merged
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--date')
|
|
p.add_argument('--limit', type=int, default=100, help='Anzahl Stories in der Ausgabe')
|
|
p.add_argument('--workers', type=int, default=8)
|
|
p.add_argument('--out', default='site/stories.json')
|
|
args = p.parse_args()
|
|
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
date = (dt.datetime.strptime(args.date, "%Y%m%d").date() if args.date
|
|
else now.date())
|
|
stamps = slots(date, until=now if date == now.date() else None)
|
|
codes = {**EUROPE, **TRANSCONTINENTAL}
|
|
|
|
print(f"{date}: {len(stamps)} Pakete")
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
ev_results = list(pool.map(fetch_slot, stamps))
|
|
world = pd.concat([df for df, _ in ev_results if df is not None],
|
|
ignore_index=True)
|
|
events = world[world['ActionGeo_CountryCode'].isin(codes)].copy()
|
|
for col in NUMERIC:
|
|
events[col] = pd.to_numeric(events[col], errors='coerce')
|
|
events['ActionGeo_Country'] = events['ActionGeo_CountryCode'].map(codes)
|
|
print(f" {len(events):,} Europa-Events aus {len(world):,} weltweit")
|
|
|
|
print(" GKG-Titel laden (das ist der grosse Teil)...")
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
maps = list(pool.map(fetch_titles, stamps))
|
|
titles = {}
|
|
for m in maps:
|
|
titles.update(m)
|
|
print(f" {len(titles):,} Titel im GKG")
|
|
|
|
stories = build_stories(events, titles, cameo_labels(), args.limit)
|
|
hit = sum(1 for s in stories if s['title'])
|
|
payload = {
|
|
'generated': now.strftime('%Y-%m-%dT%H:%M:%SZ'),
|
|
'date': str(date),
|
|
'slots': len(stamps),
|
|
'events_world': int(len(world)),
|
|
'events_europe': int(len(events)),
|
|
'articles_total': int(events['SOURCEURL'].nunique()),
|
|
'countries': events['ActionGeo_Country'].value_counts().head(14).to_dict(),
|
|
'top_codes': (events['EventCode'].map(lambda c: cameo_labels().get(str(c), ('?', None))[0])
|
|
.value_counts().head(8).to_dict()),
|
|
'stories': stories,
|
|
}
|
|
Path(args.out).write_text(json.dumps(payload, ensure_ascii=False, indent=1))
|
|
print(f" {len(stories)} Stories, davon {hit} mit Titel -> {args.out}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|