From e61a81eb9ac1c1cd3819c346e552833a34ad4b34 Mon Sep 17 00:00:00 2001
From: "Till (Cowork)" <mail@tillheidrich.de>
Date: Mon, 29 Jun 2026 12:36:57 +0000
Subject: [PATCH] =?UTF-8?q?Crawler:=20Per-Host-Proxy-Routing=20(HTTP=5FPRO?=
 =?UTF-8?q?XY=5FURL=20nur=20f=C3=BCr=20harte=20Hosts=20via=20CRAWL=5FPROXY?=
 =?UTF-8?q?=5FHOSTS,=20Rest=20direkt)=20->=20Residential-Proxy=20sp=C3=A4t?=
 =?UTF-8?q?er=20mit=20minimalem=20Traffic=20einsetzbar.=20Hinweis:=20Datac?=
 =?UTF-8?q?enter-Proxys=20(Webshare=20Free)=20knacken=20Bauhaus/Otto=20NIC?=
 =?UTF-8?q?HT?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 livecrawler/app.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/livecrawler/app.py b/livecrawler/app.py
index 5e56a59..ba15bb7 100644
--- a/livecrawler/app.py
+++ b/livecrawler/app.py
@@ -30,8 +30,11 @@ UA = {
   "sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1",
   "upgrade-insecure-requests":"1",
 }
-# Optionaler Proxy (DE-Residential empfohlen) – wie AidaBot. Env: HTTP_PROXY_URL
+# Optionaler Proxy (DE-RESIDENTIAL empfohlen – Datacenter-Proxys werden von Bauhaus/Otto geblockt!).
+# Env: HTTP_PROXY_URL. Wird NUR für die harten Hosts genutzt (CRAWL_PROXY_HOSTS), Rest läuft direkt
+# -> minimaler Proxy-Traffic (schont Bandbreiten-Limits).
 PROXY = os.getenv("HTTP_PROXY_URL","").strip() or None
+PROXY_HOSTS = [h.strip() for h in os.getenv("CRAWL_PROXY_HOSTS","bauhaus.info,otto.de,euronics.de,expert.de,amazon.de,mediamarkt.de,saturn.de").split(",") if h.strip()]
 
 def _num(s):
     s=s.replace(".","").replace("\xa0"," ")
@@ -120,13 +123,18 @@ async def crawl():
     now=datetime.now(timezone.utc).isoformat()
     sem=asyncio.Semaphore(CONCURRENCY)
     limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
-    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
+    direct=httpx.AsyncClient(headers=UA, http2=False, limits=limits)
+    proxied=httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) if PROXY else None
+    def pick(url):
+        if proxied and any(h in url for h in PROXY_HOSTS): return proxied
+        return direct
+    try:
         # Auto-Discovery: zusätzliche Produkte aus den Kategorieseiten (toom, OBI) finden
         sources=list(SOURCES); discovered=0
         if DISCOVER:
             try:
                 curated_urls={s["url"] for s in SOURCES}
-                disc=[d for d in await discover(client, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
+                disc=[d for d in await discover(direct, cap=DISCOVER_CAP) if d["url"] not in curated_urls]
                 sources+=disc; discovered=len(disc)
                 log.info("discovery: +%d Produkte aus Kategorieseiten", discovered)
             except Exception as e:
@@ -134,9 +142,12 @@ async def crawl():
         async def guarded(src):
             async with sem:
                 await asyncio.sleep(0.2)  # kleiner Jitter, trotzdem höflich
-                return await fetch_one(client, src)
+                return await fetch_one(pick(src["url"]), src)
         # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt
         results=list(await asyncio.gather(*[guarded(s) for s in sources]))
+    finally:
+        await direct.aclose()
+        if proxied: await proxied.aclose()
     # Restock-Events
     prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
     hist=load(HIST,{"events":[]})