diff --git a/livecrawler/app.py b/livecrawler/app.py
index 21ca21d..9bbcf28 100644
--- a/livecrawler/app.py
+++ b/livecrawler/app.py
@@ -17,7 +17,17 @@ logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
-UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"}
+UA = {
+ "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+ "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language":"de-DE,de;q=0.9,en;q=0.6",
+ "sec-ch-ua":'"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+ "sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',
+ "sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1",
+ "upgrade-insecure-requests":"1",
+}
+# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL
+PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None
def _num(s):
s=s.replace(".","").replace("\xa0"," ")
@@ -54,11 +64,14 @@ def parse(htmltext):
async def fetch_one(client, src):
try:
- r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
+ r= await client.get(src["url"], timeout=25, follow_redirects=True)
if r.status_code in (404,410):
# Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
"url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
+ if r.status_code in (401,403,429):
+ return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
+ "url":src["url"],"price":None,"availability":"blocked","status":r.status_code}
if r.status_code!=200:
return {**src,"price":None,"availability":"unknown","status":r.status_code}
price, avail = parse(r.text)
@@ -75,7 +88,7 @@ def load(path,default):
async def crawl():
now=datetime.now(timezone.utc).isoformat()
- async with httpx.AsyncClient() as client:
+ async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False) as client:
results=[]
for src in SOURCES:
results.append(await fetch_one(client, src))
@@ -96,7 +109,8 @@ async def crawl():
if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
gone=[o for o in results if o.get("availability")=="discontinued"]
- snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
+ blocked=[o for o in results if o.get("availability")=="blocked"]
+ snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY),
"offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
json.dump(hist, open(HIST,"w"), ensure_ascii=False)
diff --git a/src/pages/karte.astro b/src/pages/karte.astro
index 6734f10..1839186 100644
--- a/src/pages/karte.astro
+++ b/src/pages/karte.astro
@@ -220,13 +220,14 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü
if(!d||!d.offers||!d.offers.length)return;
const when=d.generatedAt?new Date(d.generatedAt):null;
const mins=when?Math.max(1,Math.round((Date.now()-when)/60000)):'?';
- const items=d.offers.filter(o=>o.price).map(o=>{
+ const items=d.offers.filter(o=>o.price||o.availability==='discontinued').map(o=>{
const av=o.availability==='in_stock'?'im Markt':(o.availability==='online_available'?'online':(o.availability==='discontinued'?'⚠ Seite entfernt (404)':'vergriffen'));
const cls=(o.availability==='out_of_stock')?'':'';
return `${o.chain} · LIVE${fmt(o.price)}${o.name} · ${av}`;
}).join('');
const bar=$('#onlinebar');
- bar.insertAdjacentHTML('afterbegin', ` LIVE gescrapt · Stand vor ${mins} Min`+items);
+ const note = d.blocked?` · ${d.blocked} Quelle(n) blockiert (Proxy nötig)`:'';
+ bar.insertAdjacentHTML('afterbegin', ` LIVE gescrapt · Stand vor ${mins} Min${note}`+items);
}).catch(()=>{});
}
map.on('load',()=>{