"""Kühlfinder Live-Crawler — eigener Verfügbarkeits-/Preis-Scraper.
Muster gespiegelt vom AIDA Preisradar (FastAPI + httpx + APScheduler + /app/data + Coolify).
Scrapt echte, server-gerenderte Händlerseiten (OBI, Bauhaus), persistiert Snapshots +
Restock-Historie, liefert /api/live (CORS). Keine Drittdaten (kein braucheklima-Import)."""
import os, re, json, html, logging, asyncio
from datetime import datetime, timezone
import httpx
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.interval import IntervalTrigger
from sources import SOURCES

logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"}

def _num(s):
    s=s.replace(".","").replace("\xa0"," ")
    m=re.search(r"(\d{2,4})[,](\d{2})",s) or re.search(r"(\d{2,4}),-",s) or re.search(r"(\d{2,4})\s*€",s)
    if not m: return None
    g=m.groups()
    return float(g[0]) + (float(g[1])/100 if len(g)>1 and g[1] else 0)

def parse(htmltext):
    # 1) JSON-LD Offer
    price=None; avail=None
    for m in re.finditer(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', htmltext, re.S|re.I):
        try: data=json.loads(m.group(1))
        except Exception: continue
        for node in (data if isinstance(data,list) else [data]):
            graph = node.get("@graph",[node]) if isinstance(node,dict) else [node]
            for g in graph:
                if not isinstance(g,dict): continue
                off=g.get("offers")
                if off:
                    off=off[0] if isinstance(off,list) else off
                    try: price=float(off.get("price") or off.get("lowPrice"))
                    except Exception: pass
                    avail=str(off.get("availability") or "")
    # 2) Heuristik
    low=htmltext.lower()
    in_store = ("vorrätig" in low or "verfügbar" in low) and not ("nicht vorrätig" in low or "nicht verfügbar" in low)
    online = ("in den warenkorb" in low) or ("lieferung" in low and "nicht möglich" not in low) or (avail and "instock" in avail.lower())
    if price is None:
        mp=_num(htmltext[:200000])
        price=mp
    availability = "in_stock" if in_store else ("online_available" if online else "out_of_stock")
    return price, availability

async def fetch_one(client, src):
    try:
        r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
        if r.status_code in (404,410):
            # Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
            return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
                    "url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
        if r.status_code!=200:
            return {**src,"price":None,"availability":"unknown","status":r.status_code}
        price, avail = parse(r.text)
        return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
                "url":src["url"],"price":price,"availability":avail}
    except Exception as e:
        log.warning("fetch fail %s: %s", src["url"], e)
        return {**src,"price":None,"availability":"unknown"}

def load(path,default):
    try:
        with open(path) as f: return json.load(f)
    except Exception: return default

async def crawl():
    now=datetime.now(timezone.utc).isoformat()
    async with httpx.AsyncClient() as client:
        results=[]
        for src in SOURCES:
            results.append(await fetch_one(client, src))
            await asyncio.sleep(1.2)  # höflich: Jitter/Rate-Limit
    # Restock-Events
    prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
    hist=load(HIST,{"events":[]})
    for o in results:
        p=prev.get((o["chain"],o["productId"]))
        if p and p.get("availability")=="out_of_stock" and o.get("availability") in ("in_stock","online_available"):
            hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now})
        if p and p.get("availability")!="discontinued" and o.get("availability")=="discontinued":
            hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now,"type":"discontinued"})
    hist["events"]=hist["events"][-200:]
    # Preis-Aggregat je Produkt
    byp={}
    for o in results:
        if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
    agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
    gone=[o for o in results if o.get("availability")=="discontinued"]
    snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
          "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
    json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
    json.dump(hist, open(HIST,"w"), ensure_ascii=False)
    log.info("crawl done: %d offers, %d events", len(results), len(hist["events"]))
    return snap

app=FastAPI(title="Kühlfinder Live-Crawler", version="0.1.0")
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"])
app.add_middleware(GZipMiddleware, minimum_size=400)
sched=AsyncIOScheduler()

@app.on_event("startup")
async def _start():
    await crawl()
    sched.add_job(crawl, IntervalTrigger(minutes=INTERVAL_MIN), id="crawl", max_instances=1, coalesce=True)
    sched.start()

@app.get("/health")
def health(): return {"ok":True}

@app.get("/api/live")
def live(): return JSONResponse(load(LIVE,{"offers":[],"note":"warming up"}))