init
This commit is contained in:
+214
@@ -0,0 +1,214 @@
|
||||
"""Baut aus den GDELT-Feeds eine story-zentrierte JSON-Datei fuer die Website.
|
||||
|
||||
python build_site.py # heute (UTC)
|
||||
python build_site.py --date 20260901 --limit 120
|
||||
|
||||
Joint die Event-Tabelle mit den Artikeltiteln aus dem GKG-Feed, verdichtet
|
||||
auf Artikel-Ebene und uebersetzt die CAMEO-Codes in Klartext.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from load_europe import EUROPE, TRANSCONTINENTAL, fetch_slot, slots
|
||||
from tracker import NUMERIC
|
||||
|
||||
GKG_URL = "http://data.gdeltproject.org/gdeltv2/{}.gkg.csv.zip"
|
||||
CAMEO_JSON = (Path(__file__).parent / '.venv' / 'lib' / 'python3.14'
|
||||
/ 'site-packages' / 'data' / 'cameoCodes.json')
|
||||
|
||||
# QuadClass: verbal/material x kooperativ/konfliktiv
|
||||
QUAD = {'1': 'verbal kooperativ', '2': 'material kooperativ',
|
||||
'3': 'verbal konfliktiv', '4': 'material konfliktiv'}
|
||||
|
||||
TITLE_RE = re.compile(r'<PAGE_TITLE>(.*?)</PAGE_TITLE>', re.S)
|
||||
|
||||
|
||||
def cameo_labels():
|
||||
"""{EventCode: (Beschreibung, GoldsteinScale)} aus dem CAMEO-Codebook."""
|
||||
raw = json.loads(CAMEO_JSON.read_text())
|
||||
df = pd.DataFrame(raw)
|
||||
return {str(k): (v['Description'], v['GoldsteinScale'])
|
||||
for k, v in df.to_dict('index').items()}
|
||||
|
||||
|
||||
def fetch_titles(stamp):
|
||||
"""{url: titel} aus einem GKG-Paket. Leer, wenn das Paket fehlt."""
|
||||
try:
|
||||
with urllib.request.urlopen(GKG_URL.format(stamp), timeout=120) as r:
|
||||
raw = r.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
if err.code == 404:
|
||||
return {}
|
||||
raise
|
||||
z = zipfile.ZipFile(io.BytesIO(raw))
|
||||
text = z.open(z.namelist()[0]).read().decode('utf-8', 'replace')
|
||||
out = {}
|
||||
for line in text.split('\n'):
|
||||
cols = line.split('\t')
|
||||
if len(cols) < 27:
|
||||
continue
|
||||
m = TITLE_RE.search(cols[-1])
|
||||
if m:
|
||||
out[cols[4]] = html_unescape(m.group(1).strip())
|
||||
return out
|
||||
|
||||
|
||||
def html_unescape(s):
|
||||
import html
|
||||
# GKG kodiert doppelt: — und &#x2014; kommen beide vor
|
||||
return html.unescape(html.unescape(s))
|
||||
|
||||
|
||||
def build_stories(events, titles, cameo, limit):
|
||||
"""Verdichtet Event-Zeilen auf Artikel-Ebene."""
|
||||
events = events.copy()
|
||||
events['title'] = events['SOURCEURL'].map(titles)
|
||||
events['domain'] = events['SOURCEURL'].str.extract(r'https?://(?:www\.)?([^/]+)')
|
||||
|
||||
stories = []
|
||||
for url, grp in events.groupby('SOURCEURL'):
|
||||
top = grp.loc[grp['NumArticles'].idxmax()]
|
||||
seen = set()
|
||||
acts = []
|
||||
for _, e in grp.sort_values('NumArticles', ascending=False).iterrows():
|
||||
desc, gold = cameo.get(str(e.EventCode), (f'CAMEO {e.EventCode}', None))
|
||||
key = (e.Actor1Name, e.Actor2Name, e.EventCode)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
acts.append({
|
||||
'code': e.EventCode, 'label': desc,
|
||||
'goldstein': None if pd.isna(e.GoldsteinScale) else float(e.GoldsteinScale),
|
||||
'quad': QUAD.get(str(e.QuadClass), ''),
|
||||
'actor1': None if pd.isna(e.Actor1Name) else e.Actor1Name.title(),
|
||||
'actor2': None if pd.isna(e.Actor2Name) else e.Actor2Name.title(),
|
||||
})
|
||||
|
||||
stories.append({
|
||||
'url': url,
|
||||
'title': top['title'] if isinstance(top['title'], str) else None,
|
||||
'domain': top['domain'],
|
||||
'country': top['ActionGeo_Country'],
|
||||
'place': top['ActionGeo_FullName'],
|
||||
'lat': None if pd.isna(top['ActionGeo_Lat']) else float(top['ActionGeo_Lat']),
|
||||
'lon': None if pd.isna(top['ActionGeo_Long']) else float(top['ActionGeo_Long']),
|
||||
'seen': str(top['DATEADDED']),
|
||||
'articles': int(grp['NumArticles'].max()),
|
||||
'mentions': int(grp['NumMentions'].max()),
|
||||
'tone': round(float(grp['AvgTone'].mean()), 2),
|
||||
'goldstein': round(float(grp['GoldsteinScale'].mean()), 2),
|
||||
'events': acts,
|
||||
'event_rows': len(grp),
|
||||
})
|
||||
|
||||
stories = cluster_by_title(stories)
|
||||
# Stories ohne Artikeltitel sind fuer Leser wertlos - der GKG-Join hat
|
||||
# sie nicht gefunden, weil der Artikel aus einem aelteren Paket stammt.
|
||||
stories = [s for s in stories if s['title']]
|
||||
stories.sort(key=lambda s: (-s['articles'], -s['mentions']))
|
||||
return stories[:limit]
|
||||
|
||||
|
||||
def norm_title(t):
|
||||
"""Titel auf Vergleichsform bringen: Kleinschreibung, ohne Satzzeichen
|
||||
und ohne den bei vielen Quellen angehaengten Blattnamen."""
|
||||
t = t.lower().split(' - ')[0].split(' | ')[0]
|
||||
return re.sub(r'[^a-z0-9]+', ' ', t).strip()
|
||||
|
||||
|
||||
def cluster_by_title(stories):
|
||||
"""Syndizierte Agenturmeldungen zusammenfassen: gleicher Titel, andere Quelle.
|
||||
|
||||
Der Cluster behaelt die staerkste Variante; die uebrigen Quellen haengen
|
||||
als 'sources' daran, ihre Events werden mit eingesammelt.
|
||||
"""
|
||||
clusters = {}
|
||||
for s in stories:
|
||||
key = norm_title(s['title']) if s['title'] else s['url']
|
||||
clusters.setdefault(key, []).append(s)
|
||||
|
||||
merged = []
|
||||
for group in clusters.values():
|
||||
group.sort(key=lambda s: -s['articles'])
|
||||
lead = dict(group[0])
|
||||
lead['sources'] = [{'domain': g['domain'], 'url': g['url']} for g in group]
|
||||
lead['event_rows'] = sum(g['event_rows'] for g in group)
|
||||
if len(group) > 1:
|
||||
seen = set()
|
||||
acts = []
|
||||
for g in group:
|
||||
for e in g['events']:
|
||||
key = (e['actor1'], e['actor2'], e['code'])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
acts.append(e)
|
||||
lead['events'] = acts
|
||||
lead['tone'] = round(sum(g['tone'] for g in group) / len(group), 2)
|
||||
merged.append(lead)
|
||||
return merged
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--date')
|
||||
p.add_argument('--limit', type=int, default=100, help='Anzahl Stories in der Ausgabe')
|
||||
p.add_argument('--workers', type=int, default=8)
|
||||
p.add_argument('--out', default='site/stories.json')
|
||||
args = p.parse_args()
|
||||
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
date = (dt.datetime.strptime(args.date, "%Y%m%d").date() if args.date
|
||||
else now.date())
|
||||
stamps = slots(date, until=now if date == now.date() else None)
|
||||
codes = {**EUROPE, **TRANSCONTINENTAL}
|
||||
|
||||
print(f"{date}: {len(stamps)} Pakete")
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
ev_results = list(pool.map(fetch_slot, stamps))
|
||||
world = pd.concat([df for df, _ in ev_results if df is not None],
|
||||
ignore_index=True)
|
||||
events = world[world['ActionGeo_CountryCode'].isin(codes)].copy()
|
||||
for col in NUMERIC:
|
||||
events[col] = pd.to_numeric(events[col], errors='coerce')
|
||||
events['ActionGeo_Country'] = events['ActionGeo_CountryCode'].map(codes)
|
||||
print(f" {len(events):,} Europa-Events aus {len(world):,} weltweit")
|
||||
|
||||
print(" GKG-Titel laden (das ist der grosse Teil)...")
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
maps = list(pool.map(fetch_titles, stamps))
|
||||
titles = {}
|
||||
for m in maps:
|
||||
titles.update(m)
|
||||
print(f" {len(titles):,} Titel im GKG")
|
||||
|
||||
stories = build_stories(events, titles, cameo_labels(), args.limit)
|
||||
hit = sum(1 for s in stories if s['title'])
|
||||
payload = {
|
||||
'generated': now.strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'date': str(date),
|
||||
'slots': len(stamps),
|
||||
'events_world': int(len(world)),
|
||||
'events_europe': int(len(events)),
|
||||
'articles_total': int(events['SOURCEURL'].nunique()),
|
||||
'countries': events['ActionGeo_Country'].value_counts().head(14).to_dict(),
|
||||
'top_codes': (events['EventCode'].map(lambda c: cameo_labels().get(str(c), ('?', None))[0])
|
||||
.value_counts().head(8).to_dict()),
|
||||
'stories': stories,
|
||||
}
|
||||
Path(args.out).write_text(json.dumps(payload, ensure_ascii=False, indent=1))
|
||||
print(f" {len(stories)} Stories, davon {hit} mit Titel -> {args.out}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user