"""Baut aus den GDELT-Feeds eine story-zentrierte JSON-Datei fuer die Website.

  python build_site.py                  # heute (UTC)
  python build_site.py --date 20260901 --limit 120

Joint die Event-Tabelle mit den Artikeltiteln aus dem GKG-Feed, verdichtet
auf Artikel-Ebene und uebersetzt die CAMEO-Codes in Klartext.
"""

import argparse
import datetime as dt
import io
import json
import re
import urllib.error
import urllib.request
import zipfile
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import pandas as pd

from load_europe import EUROPE, TRANSCONTINENTAL, fetch_slot, slots
from tracker import NUMERIC

GKG_URL = "http://data.gdeltproject.org/gdeltv2/{}.gkg.csv.zip"
CAMEO_JSON = (Path(__file__).parent / '.venv' / 'lib' / 'python3.14'
              / 'site-packages' / 'data' / 'cameoCodes.json')

# QuadClass: verbal/material x kooperativ/konfliktiv
QUAD = {'1': 'verbal kooperativ', '2': 'material kooperativ',
        '3': 'verbal konfliktiv', '4': 'material konfliktiv'}

TITLE_RE = re.compile(r'<PAGE_TITLE>(.*?)</PAGE_TITLE>', re.S)


def cameo_labels():
    """{EventCode: (Beschreibung, GoldsteinScale)} aus dem CAMEO-Codebook."""
    raw = json.loads(CAMEO_JSON.read_text())
    df = pd.DataFrame(raw)
    return {str(k): (v['Description'], v['GoldsteinScale'])
            for k, v in df.to_dict('index').items()}


def fetch_titles(stamp):
    """{url: titel} aus einem GKG-Paket. Leer, wenn das Paket fehlt."""
    try:
        with urllib.request.urlopen(GKG_URL.format(stamp), timeout=120) as r:
            raw = r.read()
    except urllib.error.HTTPError as err:
        if err.code == 404:
            return {}
        raise
    z = zipfile.ZipFile(io.BytesIO(raw))
    text = z.open(z.namelist()[0]).read().decode('utf-8', 'replace')
    out = {}
    for line in text.split('\n'):
        cols = line.split('\t')
        if len(cols) < 27:
            continue
        m = TITLE_RE.search(cols[-1])
        if m:
            out[cols[4]] = html_unescape(m.group(1).strip())
    return out


def html_unescape(s):
    import html
    # GKG kodiert doppelt: &#x2014; und &amp;#x2014; kommen beide vor
    return html.unescape(html.unescape(s))


def build_stories(events, titles, cameo, limit):
    """Verdichtet Event-Zeilen auf Artikel-Ebene."""
    events = events.copy()
    events['title'] = events['SOURCEURL'].map(titles)
    events['domain'] = events['SOURCEURL'].str.extract(r'https?://(?:www\.)?([^/]+)')

    stories = []
    for url, grp in events.groupby('SOURCEURL'):
        top = grp.loc[grp['NumArticles'].idxmax()]
        seen = set()
        acts = []
        for _, e in grp.sort_values('NumArticles', ascending=False).iterrows():
            desc, gold = cameo.get(str(e.EventCode), (f'CAMEO {e.EventCode}', None))
            key = (e.Actor1Name, e.Actor2Name, e.EventCode)
            if key in seen:
                continue
            seen.add(key)
            acts.append({
                'code': e.EventCode, 'label': desc,
                'goldstein': None if pd.isna(e.GoldsteinScale) else float(e.GoldsteinScale),
                'quad': QUAD.get(str(e.QuadClass), ''),
                'actor1': None if pd.isna(e.Actor1Name) else e.Actor1Name.title(),
                'actor2': None if pd.isna(e.Actor2Name) else e.Actor2Name.title(),
            })

        stories.append({
            'url': url,
            'title': top['title'] if isinstance(top['title'], str) else None,
            'domain': top['domain'],
            'country': top['ActionGeo_Country'],
            'place': top['ActionGeo_FullName'],
            'lat': None if pd.isna(top['ActionGeo_Lat']) else float(top['ActionGeo_Lat']),
            'lon': None if pd.isna(top['ActionGeo_Long']) else float(top['ActionGeo_Long']),
            'seen': str(top['DATEADDED']),
            'articles': int(grp['NumArticles'].max()),
            'mentions': int(grp['NumMentions'].max()),
            'tone': round(float(grp['AvgTone'].mean()), 2),
            'goldstein': round(float(grp['GoldsteinScale'].mean()), 2),
            'events': acts,
            'event_rows': len(grp),
        })

    stories = cluster_by_title(stories)
    # Stories ohne Artikeltitel sind fuer Leser wertlos - der GKG-Join hat
    # sie nicht gefunden, weil der Artikel aus einem aelteren Paket stammt.
    stories = [s for s in stories if s['title']]
    stories.sort(key=lambda s: (-s['articles'], -s['mentions']))
    return stories[:limit]


def norm_title(t):
    """Titel auf Vergleichsform bringen: Kleinschreibung, ohne Satzzeichen
    und ohne den bei vielen Quellen angehaengten Blattnamen."""
    t = t.lower().split(' - ')[0].split(' | ')[0]
    return re.sub(r'[^a-z0-9]+', ' ', t).strip()


def cluster_by_title(stories):
    """Syndizierte Agenturmeldungen zusammenfassen: gleicher Titel, andere Quelle.

    Der Cluster behaelt die staerkste Variante; die uebrigen Quellen haengen
    als 'sources' daran, ihre Events werden mit eingesammelt.
    """
    clusters = {}
    for s in stories:
        key = norm_title(s['title']) if s['title'] else s['url']
        clusters.setdefault(key, []).append(s)

    merged = []
    for group in clusters.values():
        group.sort(key=lambda s: -s['articles'])
        lead = dict(group[0])
        lead['sources'] = [{'domain': g['domain'], 'url': g['url']} for g in group]
        lead['event_rows'] = sum(g['event_rows'] for g in group)
        if len(group) > 1:
            seen = set()
            acts = []
            for g in group:
                for e in g['events']:
                    key = (e['actor1'], e['actor2'], e['code'])
                    if key not in seen:
                        seen.add(key)
                        acts.append(e)
            lead['events'] = acts
            lead['tone'] = round(sum(g['tone'] for g in group) / len(group), 2)
        merged.append(lead)
    return merged


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--date')
    p.add_argument('--limit', type=int, default=100, help='Anzahl Stories in der Ausgabe')
    p.add_argument('--workers', type=int, default=8)
    p.add_argument('--out', default='site/stories.json')
    args = p.parse_args()

    now = dt.datetime.now(dt.timezone.utc)
    date = (dt.datetime.strptime(args.date, "%Y%m%d").date() if args.date
            else now.date())
    stamps = slots(date, until=now if date == now.date() else None)
    codes = {**EUROPE, **TRANSCONTINENTAL}

    print(f"{date}: {len(stamps)} Pakete")
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        ev_results = list(pool.map(fetch_slot, stamps))
    world = pd.concat([df for df, _ in ev_results if df is not None],
                      ignore_index=True)
    events = world[world['ActionGeo_CountryCode'].isin(codes)].copy()
    for col in NUMERIC:
        events[col] = pd.to_numeric(events[col], errors='coerce')
    events['ActionGeo_Country'] = events['ActionGeo_CountryCode'].map(codes)
    print(f"  {len(events):,} Europa-Events aus {len(world):,} weltweit")

    print("  GKG-Titel laden (das ist der grosse Teil)...")
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        maps = list(pool.map(fetch_titles, stamps))
    titles = {}
    for m in maps:
        titles.update(m)
    print(f"  {len(titles):,} Titel im GKG")

    stories = build_stories(events, titles, cameo_labels(), args.limit)
    hit = sum(1 for s in stories if s['title'])
    payload = {
        'generated': now.strftime('%Y-%m-%dT%H:%M:%SZ'),
        'date': str(date),
        'slots': len(stamps),
        'events_world': int(len(world)),
        'events_europe': int(len(events)),
        'articles_total': int(events['SOURCEURL'].nunique()),
        'countries': events['ActionGeo_Country'].value_counts().head(14).to_dict(),
        'top_codes': (events['EventCode'].map(lambda c: cameo_labels().get(str(c), ('?', None))[0])
                      .value_counts().head(8).to_dict()),
        'stories': stories,
    }
    Path(args.out).write_text(json.dumps(payload, ensure_ascii=False, indent=1))
    print(f"  {len(stories)} Stories, davon {hit} mit Titel -> {args.out}")


if __name__ == '__main__':
    main()