diff --git a/livecrawler/__pycache__/discovery.cpython-310.pyc b/livecrawler/__pycache__/discovery.cpython-310.pyc new file mode 100644 index 0000000..2a87743 Binary files /dev/null and b/livecrawler/__pycache__/discovery.cpython-310.pyc differ diff --git a/livecrawler/app.py b/livecrawler/app.py index bc46baa..5e56a59 100644 --- a/livecrawler/app.py +++ b/livecrawler/app.py @@ -12,12 +12,15 @@ from fastapi.responses import JSONResponse from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.interval import IntervalTrigger from sources import SOURCES +from discovery import discover logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler") DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True) LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json") -INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","5")) -CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","6")) +INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","3")) +CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","5")) +DISCOVER = os.getenv("CRAWL_DISCOVER","1") not in ("0","false","") +DISCOVER_CAP = int(os.getenv("CRAWL_DISCOVER_CAP","20")) UA = { "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", @@ -103,7 +106,7 @@ async def fetch_one(client, src): return {**src,"price":None,"availability":"unknown","status":r.status_code} price, avail = parse(r.text) return {"chain":src["chain"],"kind":src["kind"],"cc":src.get("cc","DE"),"productId":src["productId"],"name":src["name"], - "url":src["url"],"price":price,"availability":avail} + "url":src["url"],"price":price,"availability":avail,"discovered":src.get("discovered",False)} except Exception as e: log.warning("fetch fail %s: %s", src["url"], e) return {**src,"price":None,"availability":"unknown"} @@ -118,13 +121,22 @@ async def crawl(): sem=asyncio.Semaphore(CONCURRENCY) limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY) async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client: + # Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden + sources=list(SOURCES); discovered=0 + if DISCOVER: + try: + curated_urls={s["url"] for s in SOURCES} + disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls] + sources+=disc; discovered=len(disc) + log.info("discovery: +%d Produkte aus Kategorieseiten", discovered) + except Exception as e: + log.warning("discovery failed: %s", e) async def guarded(src): async with sem: await asyncio.sleep(0.2) # kleiner Jitter, trotzdem höflich return await fetch_one(client, src) - # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für 5-Min-Takt - results=await asyncio.gather(*[guarded(s) for s in SOURCES]) - results=list(results) + # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt + results=list(await asyncio.gather(*[guarded(s) for s in sources])) # Restock-Events prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) } hist=load(HIST,{"events":[]}) @@ -143,6 +155,7 @@ async def crawl(): gone=[o for o in results if o.get("availability")=="discontinued"] blocked=[o for o in results if o.get("availability")=="blocked"] snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY), + "curatedCount":len(SOURCES),"discoveredCount":discovered,"totalSources":len(sources), "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]} json.dump(snap, open(LIVE,"w"), ensure_ascii=False) json.dump(hist, open(HIST,"w"), ensure_ascii=False) diff --git a/livecrawler/discovery.py b/livecrawler/discovery.py new file mode 100644 index 0000000..83d9418 --- /dev/null +++ b/livecrawler/discovery.py @@ -0,0 +1,50 @@ +"""Auto-Discovery: liest die Klimageräte-Kategorieseiten der scrapebaren Baumärkte +und findet Produktseiten SELBST (statt hartkodierter URLs). So wächst die Abdeckung +automatisch mit dem Sortiment. Server-gerenderte Chains: toom, OBI. +Hornbach-Listing = JS (liefert keine Links), Bauhaus = IP-Block (403) -> brauchen Proxy/Headless. +Curated SOURCES bleiben für den Cross-Shop-Preisvergleich (gleiche productId über Shops).""" +import re + +# Kategorie-Seiten + Produktlink-Muster je Chain +CATALOG = [ + {"chain":"toom","kind":"baumarkt","cc":"DE","base":"https://toom.de", + "cat":"https://toom.de/c/bauen-renovieren/klima-lueftung/klimaanlagen/klimageraete", + "re":r'/p/([a-z0-9-]+)/(\d+)', "slug_grp":1, "id_grp":2}, + {"chain":"OBI","kind":"baumarkt","cc":"DE","base":"https://www.obi.de", + "cat":"https://www.obi.de/search/klimager%C3%A4te/", + "re":r'/p/(\d+)/([a-z0-9-]+)', "slug_grp":2, "id_grp":1}, +] +# Nur echte Klimageräte behalten, Zubehör/Empfehlungs-Querlinks raus (Daten-Hygiene) +KEEP = re.compile(r'(klimage|klimaanlage|klimasplit|split-klima|portasplit|mobiles?-klima|btu|pinguino|comfee|midea|delonghi|suntec|klarstein|remko|trotec|aeg|bosch)', re.I) +SKIP = re.compile(r'(ventilator|fensterabdicht|abluftschlauch|fernbedien|ersatzfilter|halterung|wandhalter|zubeh|konvektor|heizk|luftentfeucht|luftk\w+hler|tischvent|standvent|deckenvent|abdeckhaube|abdeckung|reinig|kuehlverbind|k\w+hlverbind|kuehlleitung|\bleitung\b|\bkit\b|montage|adapter|schlauch)', re.I) + +def _name(slug): + s=re.sub(r'\b(eek|a|aplus|aplusplus|schwarz|weiss|grau|inkl|mit|und)\b',' ',slug.replace('-',' '),flags=re.I) + s=re.sub(r'\s+',' ',s).strip().title() + return s[:60] or slug + +async def discover(client, cap=20): + out=[] + for c in CATALOG: + try: + r=await client.get(c["cat"], timeout=25, follow_redirects=True) + if r.status_code!=200: + continue + seen=set(); found=0 + for m in re.finditer(c["re"], r.text): + slug=m.group(c["slug_grp"]); pid_num=m.group(c["id_grp"]) + if not KEEP.search(slug) or SKIP.search(slug): + continue + url=c["base"]+m.group(0) + if url in seen: + continue + seen.add(url) + out.append({"chain":c["chain"],"kind":c["kind"],"cc":c["cc"], + "productId":f"disc-{c['chain'].lower()}-{pid_num}", + "name":_name(slug),"url":url,"discovered":True}) + found+=1 + if found>=cap: + break + except Exception: + pass + return out