till/kuehlfinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Live-Crawler scharf: Frontend-Overlay (LIVE-gescrapte OBI/Bauhaus-Preise im Monitor) + 404/410-Erkennung (Produktseite entfernt) – Edge, den braucheklima nicht hat

Browse Source
This commit is contained in:
Till Heidrich
2026-06-28 07:39:39 +00:00
parent bca2107ad8
commit bb7991bbd3
2 changed files with 26 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livecrawler/app.py
+9 -2
View File
@@ -55,7 +55,11 @@ def parse(htmltext):
async def fetch_one(client, src):
try:
r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
if r.status_code!=200:
if r.status_code in (404,410):
# Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
"url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
if r.status_code!=200:
return {**src,"price":None,"availability":"unknown","status":r.status_code}
price, avail = parse(r.text)
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
@@ -83,13 +87,16 @@ async def crawl():
p=prev.get((o["chain"],o["productId"]))
if p and p.get("availability")=="out_of_stock" and o.get("availability") in ("in_stock","online_available"):
hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now})
if p and p.get("availability")!="discontinued" and o.get("availability")=="discontinued":
hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now,"type":"discontinued"})
hist["events"]=hist["events"][-200:]
# Preis-Aggregat je Produkt
byp={}
for o in results:
if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,
gone=[o for o in results if o.get("availability")=="discontinued"]
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
"offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
json.dump(hist, open(HIST,"w"), ensure_ascii=False)