Crawler: echter Browser-Fingerprint + Proxy-Env (HTTP_PROXY_URL, DE-Residential wie AidaBot), 403/429=blocked vs 404/410=entfernt; Frontend zeigt blockierte Quellen ehrlich an

livecrawler/app.py

@@ -17,7 +17,17 @@ logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
 DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
 LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
 INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
-UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"}
+UA = {
+  "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+  "Accept-Language":"de-DE,de;q=0.9,en;q=0.6",
+  "sec-ch-ua":'"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+  "sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',
+  "sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1",
+  "upgrade-insecure-requests":"1",
+}
+# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL
+PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None
 
 def _num(s):
     s=s.replace(".","").replace("\xa0"," ")
@@ -54,11 +64,14 @@ def parse(htmltext):
 
 async def fetch_one(client, src):
     try:
-        r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
+        r= await client.get(src["url"], timeout=25, follow_redirects=True)
         if r.status_code in (404,410):
             # Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
             return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
                     "url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
+        if r.status_code in (401,403,429):
+            return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
+                    "url":src["url"],"price":None,"availability":"blocked","status":r.status_code}
         if r.status_code!=200:
             return {**src,"price":None,"availability":"unknown","status":r.status_code}
         price, avail = parse(r.text)
@@ -75,7 +88,7 @@ def load(path,default):
 
 async def crawl():
     now=datetime.now(timezone.utc).isoformat()
-    async with httpx.AsyncClient() as client:
+    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False) as client:
         results=[]
         for src in SOURCES:
             results.append(await fetch_one(client, src))
@@ -96,7 +109,8 @@ async def crawl():
         if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
     agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
     gone=[o for o in results if o.get("availability")=="discontinued"]
-    snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
+    blocked=[o for o in results if o.get("availability")=="blocked"]
+    snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY),
           "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
     json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
     json.dump(hist, open(HIST,"w"), ensure_ascii=False)

src/pages/karte.astro

@@ -220,13 +220,14 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü
       if(!d||!d.offers||!d.offers.length)return;
       const when=d.generatedAt?new Date(d.generatedAt):null;
       const mins=when?Math.max(1,Math.round((Date.now()-when)/60000)):'?';
-      const items=d.offers.filter(o=>o.price).map(o=>{
+      const items=d.offers.filter(o=>o.price||o.availability==='discontinued').map(o=>{
         const av=o.availability==='in_stock'?'im Markt':(o.availability==='online_available'?'online':(o.availability==='discontinued'?'⚠ Seite entfernt (404)':'vergriffen'));
         const cls=(o.availability==='out_of_stock')?'':'';
         return `<a class="osrc ${o.availability==='discontinued'?'wucher':''}" href="${o.url}" target="_blank" rel="sponsored nofollow noopener"><b>${o.chain} · LIVE</b><span class="num">${fmt(o.price)}</span><small>${o.name} · ${av}</small></a>`;
       }).join('');
       const bar=$('#onlinebar');
-      bar.insertAdjacentHTML('afterbegin', `<span class="livebadge" title="eigener Live-Crawler"><span class="dot ok live"></span> LIVE gescrapt · Stand vor ${mins} Min</span>`+items);
+      const note = d.blocked?` · ${d.blocked} Quelle(n) blockiert (Proxy nötig)`:'';
+      bar.insertAdjacentHTML('afterbegin', `<span class="livebadge" title="eigener Live-Crawler"><span class="dot ok live"></span> LIVE gescrapt · Stand vor ${mins} Min${note}</span>`+items);
     }).catch(()=>{});
   }
   map.on('load',()=>{