diff --git a/livecrawler/app.py b/livecrawler/app.py index 21ca21d..9bbcf28 100644 --- a/livecrawler/app.py +++ b/livecrawler/app.py @@ -17,7 +17,17 @@ logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler") DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True) LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json") INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30")) -UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"} +UA = { + "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language":"de-DE,de;q=0.9,en;q=0.6", + "sec-ch-ua":'"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + "sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"', + "sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1", + "upgrade-insecure-requests":"1", +} +# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL +PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None def _num(s): s=s.replace(".","").replace("\xa0"," ") @@ -54,11 +64,14 @@ def parse(htmltext): async def fetch_one(client, src): try: - r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True) + r= await client.get(src["url"], timeout=25, follow_redirects=True) if r.status_code in (404,410): # Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht) return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"], "url":src["url"],"price":None,"availability":"discontinued","status":r.status_code} + if r.status_code in (401,403,429): + return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"], + "url":src["url"],"price":None,"availability":"blocked","status":r.status_code} if r.status_code!=200: return {**src,"price":None,"availability":"unknown","status":r.status_code} price, avail = parse(r.text) @@ -75,7 +88,7 @@ def load(path,default): async def crawl(): now=datetime.now(timezone.utc).isoformat() - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False) as client: results=[] for src in SOURCES: results.append(await fetch_one(client, src)) @@ -96,7 +109,8 @@ async def crawl(): if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"]) agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()} gone=[o for o in results if o.get("availability")=="discontinued"] - snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone), + blocked=[o for o in results if o.get("availability")=="blocked"] + snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY), "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]} json.dump(snap, open(LIVE,"w"), ensure_ascii=False) json.dump(hist, open(HIST,"w"), ensure_ascii=False) diff --git a/src/pages/karte.astro b/src/pages/karte.astro index 6734f10..1839186 100644 --- a/src/pages/karte.astro +++ b/src/pages/karte.astro @@ -220,13 +220,14 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü if(!d||!d.offers||!d.offers.length)return; const when=d.generatedAt?new Date(d.generatedAt):null; const mins=when?Math.max(1,Math.round((Date.now()-when)/60000)):'?'; - const items=d.offers.filter(o=>o.price).map(o=>{ + const items=d.offers.filter(o=>o.price||o.availability==='discontinued').map(o=>{ const av=o.availability==='in_stock'?'im Markt':(o.availability==='online_available'?'online':(o.availability==='discontinued'?'⚠ Seite entfernt (404)':'vergriffen')); const cls=(o.availability==='out_of_stock')?'':''; return `${o.chain} · LIVE${fmt(o.price)}${o.name} · ${av}`; }).join(''); const bar=$('#onlinebar'); - bar.insertAdjacentHTML('afterbegin', ` LIVE gescrapt · Stand vor ${mins} Min`+items); + const note = d.blocked?` · ${d.blocked} Quelle(n) blockiert (Proxy nötig)`:''; + bar.insertAdjacentHTML('afterbegin', ` LIVE gescrapt · Stand vor ${mins} Min${note}`+items); }).catch(()=>{}); } map.on('load',()=>{