From bb7991bbd33a61c6d1b1af7ff7396bb4532b8b0d Mon Sep 17 00:00:00 2001
From: "Till (Cowork)" <mail@tillheidrich.de>
Date: Sun, 28 Jun 2026 07:39:39 +0000
Subject: [PATCH] =?UTF-8?q?Live-Crawler=20scharf:=20Frontend-Overlay=20(LI?=
 =?UTF-8?q?VE-gescrapte=20OBI/Bauhaus-Preise=20im=20Monitor)=20+=20404/410?=
 =?UTF-8?q?-Erkennung=20(Produktseite=20entfernt)=20=E2=80=93=20Edge,=20de?=
 =?UTF-8?q?n=20braucheklima=20nicht=20hat?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 livecrawler/app.py    | 11 +++++++++--
 src/pages/karte.astro | 17 +++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/livecrawler/app.py b/livecrawler/app.py
index b76b912..21ca21d 100644
--- a/livecrawler/app.py
+++ b/livecrawler/app.py
@@ -55,7 +55,11 @@ def parse(htmltext):
 async def fetch_one(client, src):
     try:
         r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
-        if r.status_code!=200: 
+        if r.status_code in (404,410):
+            # Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
+            return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
+                    "url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
+        if r.status_code!=200:
             return {**src,"price":None,"availability":"unknown","status":r.status_code}
         price, avail = parse(r.text)
         return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
@@ -83,13 +87,16 @@ async def crawl():
         p=prev.get((o["chain"],o["productId"]))
         if p and p.get("availability")=="out_of_stock" and o.get("availability") in ("in_stock","online_available"):
             hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now})
+        if p and p.get("availability")!="discontinued" and o.get("availability")=="discontinued":
+            hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now,"type":"discontinued"})
     hist["events"]=hist["events"][-200:]
     # Preis-Aggregat je Produkt
     byp={}
     for o in results:
         if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
     agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
-    snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,
+    gone=[o for o in results if o.get("availability")=="discontinued"]
+    snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
           "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
     json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
     json.dump(hist, open(HIST,"w"), ensure_ascii=False)
diff --git a/src/pages/karte.astro b/src/pages/karte.astro
index ab64b53..6734f10 100644
--- a/src/pages/karte.astro
+++ b/src/pages/karte.astro
@@ -123,6 +123,7 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü
   .osrc b{font-size:.82rem}.osrc .num{font-size:.92rem}.osrc small{font-size:.71rem;color:var(--slate)}
   .osrc.wucher{border-color:var(--danger);background:var(--danger-bg)}.osrc.wucher .num{color:var(--danger)}
   .trust{font-size:.68rem;color:var(--slate)}
+  .livebadge{display:inline-flex;align-items:center;gap:.4rem;font-size:.8rem;font-weight:600;color:var(--ok-dot);background:var(--ok-bg);border:1px solid var(--ok-dot);border-radius:999px;padding:.35rem .7rem;align-self:center}
   .tablewrap{margin-top:.8rem;padding:0;overflow:hidden}
   .mtab{width:100%;border-collapse:collapse;font-size:.9rem}
   .mtab thead th{text-align:left;font-size:.72rem;text-transform:uppercase;letter-spacing:.04em;color:var(--slate);font-weight:600;padding:.7rem .8rem;border-bottom:1px solid var(--line);background:var(--card);white-space:nowrap}
@@ -142,6 +143,7 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü
 <script>
   import maplibregl from 'maplibre-gl';
   const FLAG={DE:'🇩🇪',AT:'🇦🇹',CH:'🇨🇭',NL:'🇳🇱',PL:'🇵🇱',DK:'🇩🇰'};
+  const LIVE_URL='http://vnqwgi4gq520hs1p4bsgi4w6.46.224.49.66.sslip.io';
   const fmt=(n,c)=>n==null?'–':new Intl.NumberFormat('de-DE',{style:'currency',currency:c||'EUR',maximumFractionDigits:0}).format(n);
   const $=s=>document.querySelector(s),$$=s=>Array.from(document.querySelectorAll(s));
   let DATA=null,center=null,shown=120;
@@ -211,6 +213,21 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü
     $$('#countrychips .chip').forEach(b=>b.addEventListener('click',()=>{const c=b.dataset.cc;state.countries.has(c)?(state.countries.delete(c),b.classList.remove('on')):(state.countries.add(c),b.classList.add('on'));render();}));
     if(state.model!=='all')$('#model').value=state.model;
     if(state.q)$('#q').value=state.q;
+    loadLive();
+  }
+  function loadLive(){
+    fetch(LIVE_URL+'/api/live').then(r=>r.json()).then(d=>{
+      if(!d||!d.offers||!d.offers.length)return;
+      const when=d.generatedAt?new Date(d.generatedAt):null;
+      const mins=when?Math.max(1,Math.round((Date.now()-when)/60000)):'?';
+      const items=d.offers.filter(o=>o.price).map(o=>{
+        const av=o.availability==='in_stock'?'im Markt':(o.availability==='online_available'?'online':(o.availability==='discontinued'?'⚠ Seite entfernt (404)':'vergriffen'));
+        const cls=(o.availability==='out_of_stock')?'':'';
+        return `<a class="osrc ${o.availability==='discontinued'?'wucher':''}" href="${o.url}" target="_blank" rel="sponsored nofollow noopener"><b>${o.chain} · LIVE</b><span class="num">${fmt(o.price)}</span><small>${o.name} · ${av}</small></a>`;
+      }).join('');
+      const bar=$('#onlinebar');
+      bar.insertAdjacentHTML('afterbegin', `<span class="livebadge" title="eigener Live-Crawler"><span class="dot ok live"></span> LIVE gescrapt · Stand vor ${mins} Min</span>`+items);
+    }).catch(()=>{});
   }
   map.on('load',()=>{
     map.addSource('all',{type:'geojson',data:{type:'FeatureCollection',features:[]},cluster:true,clusterRadius:46,clusterMaxZoom:9});