diff --git a/livecrawler/Dockerfile b/livecrawler/Dockerfile new file mode 100644 index 0000000..3d2f514 --- /dev/null +++ b/livecrawler/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +RUN mkdir -p /app/data +EXPOSE 8000 +ENV PYTHONPATH=/app +CMD ["uvicorn","app:app","--host","0.0.0.0","--port","8000","--loop","asyncio"] diff --git a/livecrawler/app.py b/livecrawler/app.py new file mode 100644 index 0000000..b76b912 --- /dev/null +++ b/livecrawler/app.py @@ -0,0 +1,114 @@ +"""Kühlfinder Live-Crawler — eigener Verfügbarkeits-/Preis-Scraper. +Muster gespiegelt vom AIDA Preisradar (FastAPI + httpx + APScheduler + /app/data + Coolify). +Scrapt echte, server-gerenderte Händlerseiten (OBI, Bauhaus), persistiert Snapshots + +Restock-Historie, liefert /api/live (CORS). Keine Drittdaten (kein braucheklima-Import).""" +import os, re, json, html, logging, asyncio +from datetime import datetime, timezone +import httpx +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import JSONResponse +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.interval import IntervalTrigger +from sources import SOURCES + +logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler") +DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True) +LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json") +INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30")) +UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"} + +def _num(s): + s=s.replace(".","").replace("\xa0"," ") + m=re.search(r"(\d{2,4})[,](\d{2})",s) or re.search(r"(\d{2,4}),-",s) or re.search(r"(\d{2,4})\s*€",s) + if not m: return None + g=m.groups() + return float(g[0]) + (float(g[1])/100 if len(g)>1 and g[1] else 0) + +def parse(htmltext): + # 1) JSON-LD Offer + price=None; avail=None + for m in re.finditer(r']+application/ld\+json[^>]*>(.*?)', htmltext, re.S|re.I): + try: data=json.loads(m.group(1)) + except Exception: continue + for node in (data if isinstance(data,list) else [data]): + graph = node.get("@graph",[node]) if isinstance(node,dict) else [node] + for g in graph: + if not isinstance(g,dict): continue + off=g.get("offers") + if off: + off=off[0] if isinstance(off,list) else off + try: price=float(off.get("price") or off.get("lowPrice")) + except Exception: pass + avail=str(off.get("availability") or "") + # 2) Heuristik + low=htmltext.lower() + in_store = ("vorrätig" in low or "verfügbar" in low) and not ("nicht vorrätig" in low or "nicht verfügbar" in low) + online = ("in den warenkorb" in low) or ("lieferung" in low and "nicht möglich" not in low) or (avail and "instock" in avail.lower()) + if price is None: + mp=_num(htmltext[:200000]) + price=mp + availability = "in_stock" if in_store else ("online_available" if online else "out_of_stock") + return price, availability + +async def fetch_one(client, src): + try: + r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True) + if r.status_code!=200: + return {**src,"price":None,"availability":"unknown","status":r.status_code} + price, avail = parse(r.text) + return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"], + "url":src["url"],"price":price,"availability":avail} + except Exception as e: + log.warning("fetch fail %s: %s", src["url"], e) + return {**src,"price":None,"availability":"unknown"} + +def load(path,default): + try: + with open(path) as f: return json.load(f) + except Exception: return default + +async def crawl(): + now=datetime.now(timezone.utc).isoformat() + async with httpx.AsyncClient() as client: + results=[] + for src in SOURCES: + results.append(await fetch_one(client, src)) + await asyncio.sleep(1.2) # höflich: Jitter/Rate-Limit + # Restock-Events + prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) } + hist=load(HIST,{"events":[]}) + for o in results: + p=prev.get((o["chain"],o["productId"])) + if p and p.get("availability")=="out_of_stock" and o.get("availability") in ("in_stock","online_available"): + hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now}) + hist["events"]=hist["events"][-200:] + # Preis-Aggregat je Produkt + byp={} + for o in results: + if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"]) + agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()} + snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN, + "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]} + json.dump(snap, open(LIVE,"w"), ensure_ascii=False) + json.dump(hist, open(HIST,"w"), ensure_ascii=False) + log.info("crawl done: %d offers, %d events", len(results), len(hist["events"])) + return snap + +app=FastAPI(title="Kühlfinder Live-Crawler", version="0.1.0") +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"]) +app.add_middleware(GZipMiddleware, minimum_size=400) +sched=AsyncIOScheduler() + +@app.on_event("startup") +async def _start(): + await crawl() + sched.add_job(crawl, IntervalTrigger(minutes=INTERVAL_MIN), id="crawl", max_instances=1, coalesce=True) + sched.start() + +@app.get("/health") +def health(): return {"ok":True} + +@app.get("/api/live") +def live(): return JSONResponse(load(LIVE,{"offers":[],"note":"warming up"})) diff --git a/livecrawler/requirements.txt b/livecrawler/requirements.txt new file mode 100644 index 0000000..4eb87c8 --- /dev/null +++ b/livecrawler/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.104.1 +uvicorn==0.24.0 +httpx==0.25.2 +apscheduler==3.10.4 diff --git a/livecrawler/sources.py b/livecrawler/sources.py new file mode 100644 index 0000000..56b951b --- /dev/null +++ b/livecrawler/sources.py @@ -0,0 +1,12 @@ +# Verifizierte, server-gerenderte Produkt-URLs (httpx-scrapebar, Stand 28.06.2026). +# Erweiterbar: weitere Ketten/Modelle hier eintragen. +SOURCES = [ + {"chain":"OBI","kind":"baumarkt","productId":"midea-portasplit-12000","name":"Midea PortaSplit 12.000 BTU", + "url":"https://www.obi.de/p/8620890/midea-mobile-split-klimaanlage-portasplit"}, + {"chain":"OBI","kind":"baumarkt","productId":"delonghi-pac-ex105","name":"De'Longhi PAC EX105", + "url":"https://www.obi.de/p/2892115/delonghi-mobiles-klimageraet-dl-pac-ex105-r290-eek-a-"}, + {"chain":"Bauhaus","kind":"baumarkt","productId":"midea-portasplit-12000","name":"Midea PortaSplit 12.000 BTU", + "url":"https://www.bauhaus.info/klimaanlagen/midea-klimasplitgeraet-portasplit-12000-btu/p/31934233"}, +] +# TODO (mit Keys): Amazon PA-API 5.0 (Access/Secret/Partner-Tag), eBay Browse API (OAuth-App-Token). +# TODO: Per-Markt-Verfügbarkeit über chainspez. Availability-Endpunkte (storeId) – wie braucheklima.