Crawler: Per-Host-Proxy-Routing (HTTP_PROXY_URL nur für harte Hosts via CRAWL_PROXY_HOSTS, Rest direkt) -> Residential-Proxy später mit minimalem Traffic einsetzbar. Hinweis: Datacenter-Proxys (Webshare Free) knacken Bauhaus/Otto NICHT

livecrawler/app.py

@@ -30,8 +30,11 @@ UA = {
   "sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1",
   "upgrade-insecure-requests":"1",
 }
-# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL
+# Optionaler Proxy (DE-RESIDENTIAL empfohlen – Datacenter-Proxys werden von Bauhaus/Otto geblockt!).
+# Env: HTTP_PROXY_URL. Wird NUR für die harten Hosts genutzt (CRAWL_PROXY_HOSTS), Rest läuft direkt
+# -> minimaler Proxy-Traffic (schont Bandbreiten-Limits).
 PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None
+PROXY_HOSTS = [h.strip() for h in os.getenv("CRAWL_PROXY_HOSTS","bauhaus.info,otto.de,euronics.de,expert.de,amazon.de,mediamarkt.de,saturn.de").split(",") if h.strip()]
 
 def _num(s):
     s=s.replace(".","").replace("\xa0"," ")
@@ -120,13 +123,18 @@ async def crawl():
     now=datetime.now(timezone.utc).isoformat()
     sem=asyncio.Semaphore(CONCURRENCY)
     limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
-    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
+    direct=httpx.AsyncClient(headers=UA, http2=False, limits=limits)
+    proxied=httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) if PROXY else None
+    def pick(url):
+        if proxied and any(h in url for h in PROXY_HOSTS): return proxied
+        return direct
+    try:
         # Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden
         sources=list(SOURCES); discovered=0
         if DISCOVER:
             try:
                 curated_urls={s["url"] for s in SOURCES}
-                disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
+                disc=[d for d in await discover(direct, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
                 sources+=disc; discovered=len(disc)
                 log.info("discovery: +%d Produkte aus Kategorieseiten", discovered)
             except Exception as e:
@@ -134,9 +142,12 @@ async def crawl():
         async def guarded(src):
             async with sem:
                 await asyncio.sleep(0.2)  # kleiner Jitter, trotzdem höflich
-                return await fetch_one(client, src)
+                return await fetch_one(pick(src["url"]), src)
         # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt
         results=list(await asyncio.gather(*[guarded(s) for s in sources]))
+    finally:
+        await direct.aclose()
+        if proxied: await proxied.aclose()
     # Restock-Events
     prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
     hist=load(HIST,{"events":[]})