till/kuehlfinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Crawler: echter Browser-Fingerprint + Proxy-Env (HTTP_PROXY_URL, DE-Residential wie AidaBot), 403/429=blocked vs 404/410=entfernt; Frontend zeigt blockierte Quellen ehrlich an

Browse Source
This commit is contained in:
Till Heidrich
2026-06-28 07:42:40 +00:00
parent bb7991bbd3
commit c034250bac
2 changed files with 21 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livecrawler/app.py
+18 -4
View File
@@ -17,7 +17,17 @@ logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"}
UA = {
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language":"de-DE,de;q=0.9,en;q=0.6",
"sec-ch-ua":'"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',
"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1",
"upgrade-insecure-requests":"1",
}
# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL
PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None
def _num(s):
s=s.replace(".","").replace("\xa0"," ")
@@ -54,11 +64,14 @@ def parse(htmltext):
async def fetch_one(client, src):
try:
r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
r= await client.get(src["url"], timeout=25, follow_redirects=True)
if r.status_code in (404,410):
# Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
"url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
if r.status_code in (401,403,429):
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
"url":src["url"],"price":None,"availability":"blocked","status":r.status_code}
if r.status_code!=200:
return {**src,"price":None,"availability":"unknown","status":r.status_code}
price, avail = parse(r.text)
@@ -75,7 +88,7 @@ def load(path,default):
async def crawl():
now=datetime.now(timezone.utc).isoformat()
async with httpx.AsyncClient() as client:
async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False) as client:
results=[]
for src in SOURCES:
results.append(await fetch_one(client, src))
@@ -96,7 +109,8 @@ async def crawl():
if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
gone=[o for o in results if o.get("availability")=="discontinued"]
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
blocked=[o for o in results if o.get("availability")=="blocked"]
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY),
"offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
json.dump(hist, open(HIST,"w"), ensure_ascii=False)