Crawler: Auto-Discovery der Klimageraete-Kategorieseiten (toom+OBI) findet Produkte selbst (Daten-Hygiene-Filter, Zubehoer raus); 3-Min-Takt; paralleles Crawling; discovered-Flag + Counts im Snapshot
This commit is contained in:
Binary file not shown.
+19
-6
@@ -12,12 +12,15 @@ from fastapi.responses import JSONResponse
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
from sources import SOURCES
|
||||
from discovery import discover
|
||||
|
||||
logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
|
||||
DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
|
||||
LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
|
||||
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","5"))
|
||||
CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","6"))
|
||||
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","3"))
|
||||
CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","5"))
|
||||
DISCOVER = os.getenv("CRAWL_DISCOVER","1") not in ("0","false","")
|
||||
DISCOVER_CAP = int(os.getenv("CRAWL_DISCOVER_CAP","20"))
|
||||
UA = {
|
||||
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
@@ -103,7 +106,7 @@ async def fetch_one(client, src):
|
||||
return {**src,"price":None,"availability":"unknown","status":r.status_code}
|
||||
price, avail = parse(r.text)
|
||||
return {"chain":src["chain"],"kind":src["kind"],"cc":src.get("cc","DE"),"productId":src["productId"],"name":src["name"],
|
||||
"url":src["url"],"price":price,"availability":avail}
|
||||
"url":src["url"],"price":price,"availability":avail,"discovered":src.get("discovered",False)}
|
||||
except Exception as e:
|
||||
log.warning("fetch fail %s: %s", src["url"], e)
|
||||
return {**src,"price":None,"availability":"unknown"}
|
||||
@@ -118,13 +121,22 @@ async def crawl():
|
||||
sem=asyncio.Semaphore(CONCURRENCY)
|
||||
limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
|
||||
async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
|
||||
# Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden
|
||||
sources=list(SOURCES); discovered=0
|
||||
if DISCOVER:
|
||||
try:
|
||||
curated_urls={s["url"] for s in SOURCES}
|
||||
disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
|
||||
sources+=disc; discovered=len(disc)
|
||||
log.info("discovery: +%d Produkte aus Kategorieseiten", discovered)
|
||||
except Exception as e:
|
||||
log.warning("discovery failed: %s", e)
|
||||
async def guarded(src):
|
||||
async with sem:
|
||||
await asyncio.sleep(0.2) # kleiner Jitter, trotzdem höflich
|
||||
return await fetch_one(client, src)
|
||||
# parallel (begrenzt) -> auch bei vielen Quellen schnell genug für 5-Min-Takt
|
||||
results=await asyncio.gather(*[guarded(s) for s in SOURCES])
|
||||
results=list(results)
|
||||
# parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt
|
||||
results=list(await asyncio.gather(*[guarded(s) for s in sources]))
|
||||
# Restock-Events
|
||||
prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
|
||||
hist=load(HIST,{"events":[]})
|
||||
@@ -143,6 +155,7 @@ async def crawl():
|
||||
gone=[o for o in results if o.get("availability")=="discontinued"]
|
||||
blocked=[o for o in results if o.get("availability")=="blocked"]
|
||||
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY),
|
||||
"curatedCount":len(SOURCES),"discoveredCount":discovered,"totalSources":len(sources),
|
||||
"offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
|
||||
json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
|
||||
json.dump(hist, open(HIST,"w"), ensure_ascii=False)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Auto-Discovery: liest die Klimageräte-Kategorieseiten der scrapebaren Baumärkte
|
||||
und findet Produktseiten SELBST (statt hartkodierter URLs). So wächst die Abdeckung
|
||||
automatisch mit dem Sortiment. Server-gerenderte Chains: toom, OBI.
|
||||
Hornbach-Listing = JS (liefert keine Links), Bauhaus = IP-Block (403) -> brauchen Proxy/Headless.
|
||||
Curated SOURCES bleiben für den Cross-Shop-Preisvergleich (gleiche productId über Shops)."""
|
||||
import re
|
||||
|
||||
# Kategorie-Seiten + Produktlink-Muster je Chain
|
||||
CATALOG = [
|
||||
{"chain":"toom","kind":"baumarkt","cc":"DE","base":"https://toom.de",
|
||||
"cat":"https://toom.de/c/bauen-renovieren/klima-lueftung/klimaanlagen/klimageraete",
|
||||
"re":r'/p/([a-z0-9-]+)/(\d+)', "slug_grp":1, "id_grp":2},
|
||||
{"chain":"OBI","kind":"baumarkt","cc":"DE","base":"https://www.obi.de",
|
||||
"cat":"https://www.obi.de/search/klimager%C3%A4te/",
|
||||
"re":r'/p/(\d+)/([a-z0-9-]+)', "slug_grp":2, "id_grp":1},
|
||||
]
|
||||
# Nur echte Klimageräte behalten, Zubehör/Empfehlungs-Querlinks raus (Daten-Hygiene)
|
||||
KEEP = re.compile(r'(klimage|klimaanlage|klimasplit|split-klima|portasplit|mobiles?-klima|btu|pinguino|comfee|midea|delonghi|suntec|klarstein|remko|trotec|aeg|bosch)', re.I)
|
||||
SKIP = re.compile(r'(ventilator|fensterabdicht|abluftschlauch|fernbedien|ersatzfilter|halterung|wandhalter|zubeh|konvektor|heizk|luftentfeucht|luftk\w+hler|tischvent|standvent|deckenvent|abdeckhaube|abdeckung|reinig|kuehlverbind|k\w+hlverbind|kuehlleitung|\bleitung\b|\bkit\b|montage|adapter|schlauch)', re.I)
|
||||
|
||||
def _name(slug):
|
||||
s=re.sub(r'\b(eek|a|aplus|aplusplus|schwarz|weiss|grau|inkl|mit|und)\b',' ',slug.replace('-',' '),flags=re.I)
|
||||
s=re.sub(r'\s+',' ',s).strip().title()
|
||||
return s[:60] or slug
|
||||
|
||||
async def discover(client, cap=20):
|
||||
out=[]
|
||||
for c in CATALOG:
|
||||
try:
|
||||
r=await client.get(c["cat"], timeout=25, follow_redirects=True)
|
||||
if r.status_code!=200:
|
||||
continue
|
||||
seen=set(); found=0
|
||||
for m in re.finditer(c["re"], r.text):
|
||||
slug=m.group(c["slug_grp"]); pid_num=m.group(c["id_grp"])
|
||||
if not KEEP.search(slug) or SKIP.search(slug):
|
||||
continue
|
||||
url=c["base"]+m.group(0)
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
out.append({"chain":c["chain"],"kind":c["kind"],"cc":c["cc"],
|
||||
"productId":f"disc-{c['chain'].lower()}-{pid_num}",
|
||||
"name":_name(slug),"url":url,"discovered":True})
|
||||
found+=1
|
||||
if found>=cap:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
Reference in New Issue
Block a user