Crawler: echter Browser-Fingerprint + Proxy-Env (HTTP_PROXY_URL, DE-Residential wie AidaBot), 403/429=blocked vs 404/410=entfernt; Frontend zeigt blockierte Quellen ehrlich an

2026-06-28 07:42:40 +00:00
parent bb7991bbd3
commit c034250bac
2 changed files with 21 additions and 6 deletions
@@ -220,13 +220,14 @@ const jsonld={'@context':'https://schema.org','@type':'WebApplication',name:'Kü
      if(!d||!d.offers||!d.offers.length)return;
      const when=d.generatedAt?new Date(d.generatedAt):null;
      const mins=when?Math.max(1,Math.round((Date.now()-when)/60000)):'?';
-      const items=d.offers.filter(o=>o.price).map(o=>{
+      const items=d.offers.filter(o=>o.price||o.availability==='discontinued').map(o=>{
        const av=o.availability==='in_stock'?'im Markt':(o.availability==='online_available'?'online':(o.availability==='discontinued'?'⚠ Seite entfernt (404)':'vergriffen'));
        const cls=(o.availability==='out_of_stock')?'':'';
        return `<a class="osrc ${o.availability==='discontinued'?'wucher':''}" href="${o.url}" target="_blank" rel="sponsored nofollow noopener"><b>${o.chain} · LIVE</b><span class="num">${fmt(o.price)}</span><small>${o.name} · ${av}</small></a>`;
      }).join('');
      const bar=$('#onlinebar');
-      bar.insertAdjacentHTML('afterbegin', `<span class="livebadge" title="eigener Live-Crawler"><span class="dot ok live"></span> LIVE gescrapt · Stand vor ${mins} Min</span>`+items);
+      const note = d.blocked?` · ${d.blocked} Quelle(n) blockiert (Proxy nötig)`:'';
+      bar.insertAdjacentHTML('afterbegin', `<span class="livebadge" title="eigener Live-Crawler"><span class="dot ok live"></span> LIVE gescrapt · Stand vor ${mins} Min${note}</span>`+items);
    }).catch(()=>{});
  }
  map.on('load',()=>{