Crawler umfangreicher (17 Quellen: +Hornbach De'Longhi, +toom Comfee Mobile/PAC, +OBI Suntec) + 5-Min-Takt + paralleles Crawling (Semaphore); 2 neue Geräte (Suntec Progress 12.000, Comfee PAC 12000)

livecrawler/app.py

@@ -16,7 +16,8 @@ from sources import SOURCES
 logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
 DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
 LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
-INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
+INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","5"))
+CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","6"))
 UA = {
   "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
   "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -114,11 +115,16 @@ def load(path,default):
 
 async def crawl():
     now=datetime.now(timezone.utc).isoformat()
-    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False) as client:
-        results=[]
-        for src in SOURCES:
-            results.append(await fetch_one(client, src))
-            await asyncio.sleep(1.2)  # höflich: Jitter/Rate-Limit
+    sem=asyncio.Semaphore(CONCURRENCY)
+    limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
+    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
+        async def guarded(src):
+            async with sem:
+                await asyncio.sleep(0.2)  # kleiner Jitter, trotzdem höflich
+                return await fetch_one(client, src)
+        # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für 5-Min-Takt
+        results=await asyncio.gather(*[guarded(s) for s in SOURCES])
+        results=list(results)
     # Restock-Events
     prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
     hist=load(HIST,{"events":[]})