Crawler umfangreicher (17 Quellen: +Hornbach De'Longhi, +toom Comfee Mobile/PAC, +OBI Suntec) + 5-Min-Takt + paralleles Crawling (Semaphore); 2 neue Geräte (Suntec Progress 12.000, Comfee PAC 12000)

2026-06-29 12:23:33 +00:00
1 parent 2ffafd4486
commit 1396ae50e1
@@ -16,7 +16,8 @@ from sources import SOURCES
 logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
 DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
 LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
-INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
+INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","5"))
+CONCURRENCY = int(os.getenv("CRAWL_CONCURRENCY","6"))
 UA = {
  "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
  "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -114,11 +115,16 @@ def load(path,default):

 async def crawl():
    now=datetime.now(timezone.utc).isoformat()
-    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False) as client:
-        results=[]
-        for src in SOURCES:
-            results.append(await fetch_one(client, src))
-            await asyncio.sleep(1.2)  # höflich: Jitter/Rate-Limit
+    sem=asyncio.Semaphore(CONCURRENCY)
+    limits=httpx.Limits(max_connections=CONCURRENCY, max_keepalive_connections=CONCURRENCY)
+    async with httpx.AsyncClient(proxies=PROXY, headers=UA, http2=False, limits=limits) as client:
+        async def guarded(src):
+            async with sem:
+                await asyncio.sleep(0.2)  # kleiner Jitter, trotzdem höflich
+                return await fetch_one(client, src)
+        # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für 5-Min-Takt
+        results=await asyncio.gather(*[guarded(s) for s in SOURCES])
+        results=list(results)
    # Restock-Events
    prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
    hist=load(HIST,{"events":[]})
@@ -27,6 +27,15 @@ SOURCES = [
     "url":"https://www.hornbach.de/p/klimasplitgeraet-midea-portasplit-12-000-btu-105-m-weiss/12356554/"},
    {"chain":"OBI","kind":"baumarkt","productId":DL,"name":"De'Longhi PAC EX105",
     "url":"https://www.obi.de/p/2892115/delonghi-mobiles-klimageraet-dl-pac-ex105-r290-eek-a-"},
+    {"chain":"Hornbach","kind":"baumarkt","productId":DL,"name":"De'Longhi Pinguino PAC EX105",
+     "url":"https://www.hornbach.de/p/klimageraet-delonghi-pinguino-pac-ex105-100-m-350-m-h-weiss/12721730/"},
+    # --- breitere Modell-Abdeckung (verifizierte, server-gerenderte Produktseiten) ---
+    {"chain":"toom","kind":"baumarkt","productId":"comfee-mobile-12000","name":"Comfee Mobile 12000",
+     "url":"https://toom.de/p/mobiles-klimageraet-mobile-12000-12000-btuh/9350652"},
+    {"chain":"toom","kind":"baumarkt","productId":"comfee-pac-12000","name":"Comfee PAC 12000",
+     "url":"https://toom.de/p/mobiles-klimageraet-pac-12000-12000-btuh/9350584"},
+    {"chain":"OBI","kind":"baumarkt","productId":"suntec-progress-12000","name":"Suntec Progress 12.000",
+     "url":"https://www.obi.de/p/3625753/suntec-klimaanlage-progress-12-000-schwarz-eek-a"},
    # --- Alternative (Prosatech empfiehlt diese ähnlich zur PortaSplit) ---
    {"chain":"Prosatech","kind":"fachhandel","productId":CB,"name":"Fischer ClimaButler 2 3,5 kW","official":True,
     "url":"https://prosatech.de/Mobile-Klimaanlage-ClimaButler-2-35-kW"},
@@ -1 +1 @@
-{"generatedAt":"2026-06-28T12:30:00","source":"seed-demo-coverage","note":"Standort-Abdeckung als Demo. KEINE verifizierten Filialbestände. Verifizierte Live-Preise nur via Crawler (/api/live).","stats":{"storesTotal":1531,"countries":6,"baumarkt":909,"elektro":622,"updated":"12:30"},"feed":[],"online":[],"priceOverview":{"midea-portasplit-12000":{"uvp":899,"min":809,"avg":899,"max":953,"ebayLow":683,"ebayHigh":1753,"refurb":737,"wucherFrom":1169},"midea-portasplit-cool-8000":{"uvp":749,"min":674,"avg":749,"max":794,"ebayLow":569,"ebayHigh":1461,"refurb":614,"wucherFrom":974},"delonghi-pac-ex105":{"uvp":699,"min":629,"avg":699,"max":741,"ebayLow":531,"ebayHigh":1363,"refurb":573,"wucherFrom":909},"bosch-cool-5000":{"uvp":799,"min":719,"avg":799,"max":847,"ebayLow":607,"ebayHigh":1558,"refurb":655,"wucherFrom":1039},"aeg-comfort-6000":{"uvp":649,"min":584,"avg":649,"max":688,"ebayLow":493,"ebayHigh":1266,"refurb":532,"wucherFrom":844},"dreo-318s":{"uvp":429,"min":386,"avg":429,"max":455,"ebayLow":326,"ebayHigh":837,"refurb":352,"wucherFrom":558},"midea-silent-cool-26-pro-wf":{"uvp":499,"min":449,"avg":499,"max":529,"ebayLow":379,"ebayHigh":973,"refurb":409,"wucherFrom":649},"comfee-mobile-12000":{"uvp":379,"min":341,"avg":379,"max":402,"ebayLow":288,"ebayHigh":739,"refurb":311,"wucherFrom":493},"remko-rkl-495-dc":{"uvp":1599,"min":1439,"avg":1599,"max":1695,"ebayLow":1215,"ebayHigh":3118,"refurb":1311,"wucherFrom":2079},"fischer-climabutler-2":{"uvp":1249,"min":1124,"avg":1249,"max":1324,"ebayLow":949,"ebayHigh":2436,"refurb":1024,"wucherFrom":1624}},"products":{"midea-portasplit-12000":{"name":"Midea PortaSplit 12.000 BTU","uvp":899,"type":"mobile_split","apartment":true},"midea-portasplit-cool-8000":{"name":"Midea PortaSplit Cool 8.000 BTU","uvp":749,"type":"mobile_split","apartment":true},"delonghi-pac-ex105":{"name":"De'Longhi Pinguino PAC EX105","uvp":699,"type":"monoblock","apartment":true},"bosch-cool-5000":{"name":"Bosch Cool 5000","uvp":799,"type":"monoblock","apartment":true},"aeg-comfort-6000":{"name":"AEG Comfort 6000","uvp":649,"type":"monoblock","apartment":true},"dreo-318s":{"name":"Dreo 318S","uvp":429,"type":"monoblock","apartment":true},"midea-silent-cool-26-pro-wf":{"name":"Midea Silent Cool 26 Pro WF","uvp":499,"type":"monoblock","apartment":true},"comfee-mobile-12000":{"name":"Comfee Mobile 12000","uvp":379,"type":"monoblock","apartment":true},"remko-rkl-495-dc":{"name":"Remko RKL 495 DC","uvp":1599,"type":"mobile_split","apartment":false},"fischer-climabutler-2":{"name":"Fischer ClimaButler 2","uvp":1249,"type":"mobile_split","apartment":true}},"stores":[{"id":"de-expert-brand-b8c5a","c":"expert","kind":"elektro","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"expert Brand","ci":"Brand","z":"95682","la":49.9563,"lo":11.9012,"sourceMode":"seed","models":[{"id":"bosch-cool-5000","name":"Bosch Cool 5000","a":0,"p":739},{"id":"comfee-mobile-12000","name":"Comfee Mobile 12000","a":0,"p":377},{"id":"delonghi-pac-ex105","name":"De'Longhi Pinguino PAC EX105","a":0,"p":731},{"id":"dreo-318s","name":"Dreo 318S","a":0,"p":459}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-toom-brand-33589","c":"Toom","kind":"baumarkt","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Toom Brand","ci":"Brand","z":"95682","la":49.9518,"lo":11.903,"sourceMode":"seed","models":[{"id":"delonghi-pac-ex105","name":"De'Longhi Pinguino PAC EX105","a":0,"p":724},{"id":"midea-portasplit-cool-8000","name":"Midea PortaSplit Cool 8.000 BTU","a":0,"p":792}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-euronics-brand-51601","c":"Euronics","kind":"elektro","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Euronics Brand","ci":"Brand","z":"95682","la":49.969,"lo":11.9202,"sourceMode":"seed","models":[{"id":"aeg-comfort-6000","name":"AEG Comfort 6000","a":0,"p":643},{"id":"bosch-cool-5000","name":"Bosch Cool 5000","a":0,"p":772}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-globus-lüder-bcfcd","c":"Globus","kind":"baumarkt","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Globus Lüder","ci":"Lüder","z":"29394","la":52.8068,"lo":10.6683,"sourceMode":"seed","models":[{"id":"delonghi-pac-ex105","name":"De'Longhi Pinguino PAC EX105","a":0,"p":732},{"id":"midea-portasplit-12000","name":"Midea PortaSplit 12.000 BTU","a":0,"p":918},{"id":"midea-silent-cool-26-pro-wf","name":"Midea Silent Cool 26 Pro WF","a":0,"p":472},{"id":"comfee-mobile-12000","name":"Comfee Mobile 12000","a":0,"p":366}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-bauhaus-lüder-0866d","c":"Bauhaus","kind":"baumarkt","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Bauhaus Lüder","ci":"Lüder","z":"29394","la":52.8138,"lo":10.6646,"sourceMode":"seed","models":[{"id":"midea-portasplit-cool-8000","name":"Midea PortaSplit Cool 8.000 BTU","a":0,"p":707},{"id":"midea-silent-cool-26-pro-wf","name":"Midea Silent Cool 26 Pro WF","a":0,"p":524}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-mediamarkt-janneby-bb13a","c":"MediaMarkt","kind":"elektro","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"MediaMarktLine truncated
+{"generatedAt":"2026-06-28T12:30:00","source":"seed-demo-coverage","note":"Standort-Abdeckung als Demo. KEINE verifizierten Filialbestände. Verifizierte Live-Preise nur via Crawler (/api/live).","stats":{"storesTotal":1531,"countries":6,"baumarkt":909,"elektro":622,"updated":"12:30"},"feed":[],"online":[],"priceOverview":{"midea-portasplit-12000":{"uvp":899,"min":809,"avg":899,"max":953,"ebayLow":683,"ebayHigh":1753,"refurb":737,"wucherFrom":1169},"midea-portasplit-cool-8000":{"uvp":749,"min":674,"avg":749,"max":794,"ebayLow":569,"ebayHigh":1461,"refurb":614,"wucherFrom":974},"delonghi-pac-ex105":{"uvp":699,"min":629,"avg":699,"max":741,"ebayLow":531,"ebayHigh":1363,"refurb":573,"wucherFrom":909},"bosch-cool-5000":{"uvp":799,"min":719,"avg":799,"max":847,"ebayLow":607,"ebayHigh":1558,"refurb":655,"wucherFrom":1039},"aeg-comfort-6000":{"uvp":649,"min":584,"avg":649,"max":688,"ebayLow":493,"ebayHigh":1266,"refurb":532,"wucherFrom":844},"dreo-318s":{"uvp":429,"min":386,"avg":429,"max":455,"ebayLow":326,"ebayHigh":837,"refurb":352,"wucherFrom":558},"midea-silent-cool-26-pro-wf":{"uvp":499,"min":449,"avg":499,"max":529,"ebayLow":379,"ebayHigh":973,"refurb":409,"wucherFrom":649},"comfee-mobile-12000":{"uvp":379,"min":341,"avg":379,"max":402,"ebayLow":288,"ebayHigh":739,"refurb":311,"wucherFrom":493},"remko-rkl-495-dc":{"uvp":1599,"min":1439,"avg":1599,"max":1695,"ebayLow":1215,"ebayHigh":3118,"refurb":1311,"wucherFrom":2079},"fischer-climabutler-2":{"uvp":1249,"min":1124,"avg":1249,"max":1324,"ebayLow":949,"ebayHigh":2436,"refurb":1024,"wucherFrom":1624},"suntec-progress-12000":{"uvp":589,"min":530,"avg":589,"max":624,"ebayLow":448,"ebayHigh":1149,"refurb":483,"wucherFrom":766},"comfee-pac-12000":{"uvp":399,"min":359,"avg":399,"max":423,"ebayLow":303,"ebayHigh":778,"refurb":327,"wucherFrom":519}},"products":{"midea-portasplit-12000":{"name":"Midea PortaSplit 12.000 BTU","uvp":899,"type":"mobile_split","apartment":true},"midea-portasplit-cool-8000":{"name":"Midea PortaSplit Cool 8.000 BTU","uvp":749,"type":"mobile_split","apartment":true},"delonghi-pac-ex105":{"name":"De'Longhi Pinguino PAC EX105","uvp":699,"type":"monoblock","apartment":true},"bosch-cool-5000":{"name":"Bosch Cool 5000","uvp":799,"type":"monoblock","apartment":true},"aeg-comfort-6000":{"name":"AEG Comfort 6000","uvp":649,"type":"monoblock","apartment":true},"dreo-318s":{"name":"Dreo 318S","uvp":429,"type":"monoblock","apartment":true},"midea-silent-cool-26-pro-wf":{"name":"Midea Silent Cool 26 Pro WF","uvp":499,"type":"monoblock","apartment":true},"comfee-mobile-12000":{"name":"Comfee Mobile 12000","uvp":379,"type":"monoblock","apartment":true},"remko-rkl-495-dc":{"name":"Remko RKL 495 DC","uvp":1599,"type":"mobile_split","apartment":false},"fischer-climabutler-2":{"name":"Fischer ClimaButler 2","uvp":1249,"type":"mobile_split","apartment":true},"suntec-progress-12000":{"name":"Suntec Progress 12.000","uvp":589,"type":"monoblock","apartment":true},"comfee-pac-12000":{"name":"Comfee PAC 12000","uvp":399,"type":"monoblock","apartment":true}},"stores":[{"id":"de-expert-brand-b8c5a","c":"expert","kind":"elektro","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"expert Brand","ci":"Brand","z":"95682","la":49.9563,"lo":11.9012,"sourceMode":"seed","models":[{"id":"bosch-cool-5000","name":"Bosch Cool 5000","a":0,"p":739},{"id":"comfee-mobile-12000","name":"Comfee Mobile 12000","a":0,"p":377},{"id":"delonghi-pac-ex105","name":"De'Longhi Pinguino PAC EX105","a":0,"p":731},{"id":"dreo-318s","name":"Dreo 318S","a":0,"p":459}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-toom-brand-33589","c":"Toom","kind":"baumarkt","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Toom Brand","ci":"Brand","z":"95682","la":49.9518,"lo":11.903,"sourceMode":"seed","models":[{"id":"delonghi-pac-ex105","name":"De'Longhi Pinguino PAC EX105","a":0,"p":724},{"id":"midea-portasplit-cool-8000","name":"Midea PortaSplit Cool 8.000 BTU","a":0,"p":792}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-euronics-brand-51601","c":"Euronics","kind":"elektro","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Euronics Brand","ci":"Brand","z":"95682","la":49.969,"lo":11.9202,"sourceMode":"seed","models":[{"id":"aeg-comfort-6000","name":"AEG Comfort 6000","a":0,"p":643},{"id":"bosch-cool-5000","name":"Bosch Cool 5000","a":0,"p":772}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-globus-lüder-bcfcd","c":"Globus","kind":"baumarkt","cc":"DE","cur":"EUR","flag":"🇩🇪","n":"Globus Lüder","ci":"Lüder","z":"29394","la":52.8068,"lo":10.6683,"sourceMode":"seed","models":[{"id":"delonghi-pac-ex105","name":"De'Longhi Pinguino PAC EX105","a":0,"p":732},{"id":"midea-portasplit-12000","name":"Midea PortaSplit 12.000 BTU","a":0,"p":918},{"id":"midea-silent-cool-26-pro-wf","name":"Midea Silent Cool 26 Pro WF","a":0,"p":472},{"id":"comfee-mobile-12000","name":"Comfee Mobile 12000","a":0,"p":366}],"any":{"a":0,"n":0,"minP":null}},{"id":"de-bauhaus-lüder-0866d","c":"Bauhaus","kind":"baumarkt","cc":"DE","cur":"EUR","flag":"🇩🇪Line truncated
@@ -462,5 +462,99 @@
        "url": "https://prosatech.de/Mobile-Klimaanlage-ClimaButler-2-35-kW"
      }
    ]
+  },
+  {
+    "id": "suntec-progress-12000",
+    "slug": "suntec-progress-12000",
+    "name": "Suntec Progress 12.000",
+    "brand": "Suntec",
+    "type": "monoblock",
+    "tier": "B",
+    "badge": "Günstig & verbreitet",
+    "apartmentSuitable": true,
+    "shortVerdict": "Solides Monoblock-Gerät mit 12.000 BTU für Räume bis ~60 m² – oft im Baumarkt verfügbar, gutes Preis-Leistungs-Verhältnis.",
+    "coolingKw": 3.5,
+    "btuh": 12000,
+    "efficiencyCooling": "A",
+    "noiseNote": "ca. 65 dB(A)",
+    "roomSizeM2": [
+      25,
+      60
+    ],
+    "uvp": 589,
+    "fairPriceMax": 620,
+    "bestFor": [
+      "kleines Budget",
+      "schnelle Verfügbarkeit im Markt",
+      "größere Räume bis 60 m²"
+    ],
+    "notFor": [
+      "wer es besonders leise braucht",
+      "Dauerbetrieb im Schlafzimmer"
+    ],
+    "pros": [
+      "günstig",
+      "oft sofort im Baumarkt",
+      "R290-Kältemittel",
+      "3in1 Kühlen/Entfeuchten/Ventilieren"
+    ],
+    "cons": [
+      "mit 65 dB relativ laut",
+      "Monoblock (Abluftschlauch nötig)"
+    ],
+    "score": 74,
+    "sources": [
+      {
+        "label": "OBI Suntec Progress 12.000",
+        "url": "https://www.obi.de/p/3625753/suntec-klimaanlage-progress-12-000-schwarz-eek-a"
+      }
+    ]
+  },
+  {
+    "id": "comfee-pac-12000",
+    "slug": "comfee-pac-12000",
+    "name": "Comfee PAC 12000",
+    "brand": "Comfee",
+    "type": "monoblock",
+    "tier": "B",
+    "badge": "Preis-Tipp",
+    "apartmentSuitable": true,
+    "shortVerdict": "Günstiges Monoblock-Klimagerät mit 12.000 BTU, Timer und Schlafmodus – guter Einstieg für mittelgroße Räume.",
+    "coolingKw": 3.5,
+    "btuh": 12000,
+    "efficiencyCooling": "A",
+    "noiseNote": "ca. 52 dB(A)",
+    "roomSizeM2": [
+      20,
+      43
+    ],
+    "uvp": 399,
+    "fairPriceMax": 430,
+    "bestFor": [
+      "kleines Budget",
+      "mittelgroße Räume",
+      "Einsteiger"
+    ],
+    "notFor": [
+      "sehr große Räume",
+      "wer Split-Komfort will"
+    ],
+    "pros": [
+      "sehr günstig",
+      "Timer & Schlafmodus",
+      "R290",
+      "oft im Baumarkt verfügbar"
+    ],
+    "cons": [
+      "Monoblock (Abluftschlauch)",
+      "Ausstattung einfach"
+    ],
+    "score": 72,
+    "sources": [
+      {
+        "label": "toom Comfee PAC 12000",
+        "url": "https://toom.de/p/mobiles-klimageraet-pac-12000-12000-btuh/9350584"
+      }
+    ]
  }
 ]