till/kuehlfinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Crawler: Per-Host-Proxy-Routing (HTTP_PROXY_URL nur für harte Hosts via CRAWL_PROXY_HOSTS, Rest direkt) -> Residential-Proxy später mit minimalem Traffic einsetzbar. Hinweis: Datacenter-Proxys (Webshare Free) knacken Bauhaus/Otto NICHT

Browse Source
This commit is contained in:
Till Heidrich
2026-06-29 12:36:57 +00:00
parent 6b127737ac
commit e61a81eb9a
1 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livecrawler/app.py
+15 -4
View File
@@ -30,8 +30,11 @@ UA = {
"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1", "sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1",
"upgrade-insecure-requests":"1", "upgrade-insecure-requests":"1",
} }
# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL # Optionaler Proxy (DE-RESIDENTIAL empfohlen – Datacenter-Proxys werden von Bauhaus/Otto geblockt!).
# Env: HTTP_PROXY_URL. Wird NUR für die harten Hosts genutzt (CRAWL_PROXY_HOSTS), Rest läuft direkt
# -> minimaler Proxy-Traffic (schont Bandbreiten-Limits).
PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None
PROXY_HOSTS = [h.strip() for h in os.getenv("CRAWL_PROXY_HOSTS","bauhaus.info,otto.de,euronics.de,expert.de,amazon.de,mediamarkt.de,saturn.de").split(",") if h.strip()]
def _num(s): def _num(s):
s=s.replace(".","").replace("\xa0"," ") s=s.replace(".","").replace("\xa0"," ")
@@ -120,13 +123,18 @@ async def crawl():
now=datetime.now(timezone.utc).isoformat() now=datetime.now(timezone.utc).isoformat()
sem=asyncio.Semaphore(CONCURRENCY) sem=asyncio.Semaphore(CONCURRENCY)
limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY) limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client: direct=httpx.AsyncClient(headers=UA, http2=False, limits=limits)
proxied=httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) if PROXY else None
def pick(url):
if proxied and any(h in url for h in PROXY_HOSTS): return proxied
return direct
try:
# Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden # Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden
sources=list(SOURCES); discovered=0 sources=list(SOURCES); discovered=0
if DISCOVER: if DISCOVER:
try: try:
curated_urls={s["url"] for s in SOURCES} curated_urls={s["url"] for s in SOURCES}
disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls] disc=[d for d in await discover(direct, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
sources+=disc; discovered=len(disc) sources+=disc; discovered=len(disc)
log.info("discovery: +%d Produkte aus Kategorieseiten", discovered) log.info("discovery: +%d Produkte aus Kategorieseiten", discovered)
except Exception as e: except Exception as e:
@@ -134,9 +142,12 @@ async def crawl():
async def guarded(src): async def guarded(src):
async with sem: async with sem:
await asyncio.sleep(0.2) # kleiner Jitter, trotzdem höflich await asyncio.sleep(0.2) # kleiner Jitter, trotzdem höflich
return await fetch_one(client, src) return await fetch_one(pick(src["url"]), src)
# parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt
results=list(await asyncio.gather(*[guarded(s) for s in sources])) results=list(await asyncio.gather(*[guarded(s) for s in sources]))
finally:
await direct.aclose()
if proxied: await proxied.aclose()
# Restock-Events # Restock-Events
prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) } prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
hist=load(HIST,{"events":[]}) hist=load(HIST,{"events":[]})