till/kuehlfinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Crawler: Auto-Discovery der Klimageraete-Kategorieseiten (toom+OBI) findet Produkte selbst (Daten-Hygiene-Filter, Zubehoer raus); 3-Min-Takt; paralleles Crawling; discovered-Flag + Counts im Snapshot

Browse Source
This commit is contained in:
Till Heidrich
2026-06-29 12:29:14 +00:00
parent 1396ae50e1
commit 6b127737ac
3 changed files with 69 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livecrawler/app.py
+19 -6
View File
@@ -12,12 +12,15 @@ from fastapi.responses import JSONResponse
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.interval import IntervalTrigger
from sources import SOURCES
from discovery import discover
logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","5"))
CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","6"))
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","3"))
CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","5"))
DISCOVER = os.getenv("CRAWL_DISCOVER","1") not in ("0","false","")
DISCOVER_CAP = int(os.getenv("CRAWL_DISCOVER_CAP","20"))
UA = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -103,7 +106,7 @@ async def fetch_one(client, src):
return {**src,"price":None,"availability":"unknown","status":r.status_code}
price, avail = parse(r.text)
return {"chain":src["chain"],"kind":src["kind"],"cc":src.get("cc","DE"),"productId":src["productId"],"name":src["name"],
"url":src["url"],"price":price,"availability":avail}
"url":src["url"],"price":price,"availability":avail,"discovered":src.get("discovered",False)}
except Exception as e:
log.warning("fetch fail %s: %s", src["url"], e)
return {**src,"price":None,"availability":"unknown"}
@@ -118,13 +121,22 @@ async def crawl():
sem=asyncio.Semaphore(CONCURRENCY)
limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
# Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden
sources=list(SOURCES); discovered=0
if DISCOVER:
try:
curated_urls={s["url"] for s in SOURCES}
disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
sources+=disc; discovered=len(disc)
log.info("discovery: +%d Produkte aus Kategorieseiten", discovered)
except Exception as e:
log.warning("discovery failed: %s", e)
async def guarded(src):
async with sem:
await asyncio.sleep(0.2) # kleiner Jitter, trotzdem höflich
return await fetch_one(client, src)
# parallel (begrenzt) -> auch bei vielen Quellen schnell genug für 5-Min-Takt
results=await asyncio.gather(*[guarded(s) for s in SOURCES])
results=list(results)
# parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt
results=list(await asyncio.gather(*[guarded(s) for s in sources]))
# Restock-Events
prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
hist=load(HIST,{"events":[]})
@@ -143,6 +155,7 @@ async def crawl():
gone=[o for o in results if o.get("availability")=="discontinued"]
blocked=[o for o in results if o.get("availability")=="blocked"]
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY),
"curatedCount":len(SOURCES),"discoveredCount":discovered,"totalSources":len(sources),
"offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
json.dump(hist, open(HIST,"w"), ensure_ascii=False)