Crawler: Auto-Discovery der Klimageraete-Kategorieseiten (toom+OBI) findet Produkte selbst (Daten-Hygiene-Filter, Zubehoer raus); 3-Min-Takt; paralleles Crawling; discovered-Flag + Counts im Snapshot

livecrawler/app.py

@@ -12,12 +12,15 @@ from fastapi.responses import JSONResponse
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.interval import IntervalTrigger
 from sources import SOURCES
+from discovery import discover
 
 logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
 DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
 LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
-INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","5"))
+INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","3"))
-CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","6"))
+CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","5"))
+DISCOVER = os.getenv("CRAWL_DISCOVER","1") not in ("0","false","")
+DISCOVER_CAP = int(os.getenv("CRAWL_DISCOVER_CAP","20"))
 UA = {
   "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
   "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -103,7 +106,7 @@ async def fetch_one(client, src):
             return {**src,"price":None,"availability":"unknown","status":r.status_code}
         price, avail = parse(r.text)
         return {"chain":src["chain"],"kind":src["kind"],"cc":src.get("cc","DE"),"productId":src["productId"],"name":src["name"],
-                "url":src["url"],"price":price,"availability":avail}
+                "url":src["url"],"price":price,"availability":avail,"discovered":src.get("discovered",False)}
     except Exception as e:
         log.warning("fetch fail %s: %s", src["url"], e)
         return {**src,"price":None,"availability":"unknown"}
@@ -118,13 +121,22 @@ async def crawl():
     sem=asyncio.Semaphore(CONCURRENCY)
     limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
     async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
+        # Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden
+        sources=list(SOURCES); discovered=0
+        if DISCOVER:
+            try:
+                curated_urls={s["url"] for s in SOURCES}
+                disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
+                sources+=disc; discovered=len(disc)
+                log.info("discovery: +%d Produkte aus Kategorieseiten", discovered)
+            except Exception as e:
+                log.warning("discovery failed: %s", e)
         async def guarded(src):
             async with sem:
                 await asyncio.sleep(0.2)  # kleiner Jitter, trotzdem höflich
                 return await fetch_one(client, src)
-        # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für 5-Min-Takt
+        # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt
-        results=await asyncio.gather(*[guarded(s) for s in SOURCES])
+        results=list(await asyncio.gather(*[guarded(s) for s in sources]))
-        results=list(results)
     # Restock-Events
     prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
     hist=load(HIST,{"events":[]})
@@ -143,6 +155,7 @@ async def crawl():
     gone=[o for o in results if o.get("availability")=="discontinued"]
     blocked=[o for o in results if o.get("availability")=="blocked"]
     snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY),
+          "curatedCount":len(SOURCES),"discoveredCount":discovered,"totalSources":len(sources),
           "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
     json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
     json.dump(hist, open(HIST,"w"), ensure_ascii=False)

livecrawler/discovery.py

@@ -0,0 +1,50 @@
+"""Auto-Discovery: liest die Klimageräte-Kategorieseiten der scrapebaren Baumärkte
+und findet Produktseiten SELBST (statt hartkodierter URLs). So wächst die Abdeckung
+automatisch mit dem Sortiment. Server-gerenderte Chains: toom, OBI.
+Hornbach-Listing = JS (liefert keine Links), Bauhaus = IP-Block (403) -> brauchen Proxy/Headless.
+Curated SOURCES bleiben für den Cross-Shop-Preisvergleich (gleiche productId über Shops)."""
+import re
+
+# Kategorie-Seiten + Produktlink-Muster je Chain
+CATALOG = [
+    {"chain":"toom","kind":"baumarkt","cc":"DE","base":"https://toom.de",
+     "cat":"https://toom.de/c/bauen-renovieren/klima-lueftung/klimaanlagen/klimageraete",
+     "re":r'/p/([a-z0-9-]+)/(\d+)', "slug_grp":1, "id_grp":2},
+    {"chain":"OBI","kind":"baumarkt","cc":"DE","base":"https://www.obi.de",
+     "cat":"https://www.obi.de/search/klimager%C3%A4te/",
+     "re":r'/p/(\d+)/([a-z0-9-]+)', "slug_grp":2, "id_grp":1},
+]
+# Nur echte Klimageräte behalten, Zubehör/Empfehlungs-Querlinks raus (Daten-Hygiene)
+KEEP = re.compile(r'(klimage|klimaanlage|klimasplit|split-klima|portasplit|mobiles?-klima|btu|pinguino|comfee|midea|delonghi|suntec|klarstein|remko|trotec|aeg|bosch)', re.I)
+SKIP = re.compile(r'(ventilator|fensterabdicht|abluftschlauch|fernbedien|ersatzfilter|halterung|wandhalter|zubeh|konvektor|heizk|luftentfeucht|luftk\w+hler|tischvent|standvent|deckenvent|abdeckhaube|abdeckung|reinig|kuehlverbind|k\w+hlverbind|kuehlleitung|\bleitung\b|\bkit\b|montage|adapter|schlauch)', re.I)
+
+def _name(slug):
+    s=re.sub(r'\b(eek|a|aplus|aplusplus|schwarz|weiss|grau|inkl|mit|und)\b',' ',slug.replace('-',' '),flags=re.I)
+    s=re.sub(r'\s+',' ',s).strip().title()
+    return s[:60] or slug
+
+async def discover(client, cap=20):
+    out=[]
+    for c in CATALOG:
+        try:
+            r=await client.get(c["cat"], timeout=25, follow_redirects=True)
+            if r.status_code!=200:
+                continue
+            seen=set(); found=0
+            for m in re.finditer(c["re"], r.text):
+                slug=m.group(c["slug_grp"]); pid_num=m.group(c["id_grp"])
+                if not KEEP.search(slug) or SKIP.search(slug):
+                    continue
+                url=c["base"]+m.group(0)
+                if url in seen:
+                    continue
+                seen.add(url)
+                out.append({"chain":c["chain"],"kind":c["kind"],"cc":c["cc"],
+                            "productId":f"disc-{c['chain'].lower()}-{pid_num}",
+                            "name":_name(slug),"url":url,"discovered":True})
+                found+=1
+                if found>=cap:
+                    break
+        except Exception:
+            pass
+    return out