till/kuehlfinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Crawler-Parser: nur strukturierte Preise (JSON-LD/og/itemprop), loose Regex raus + Plausibilitäts-Floor 150€ -> keine Müll-Preise (expert 24€/Amazon 200€ etc.)

Browse Source
This commit is contained in:
Till Heidrich
2026-06-28 16:45:05 +00:00
parent b90133f773
commit 1ed5a7bebe
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livecrawler/app.py
+8 -3
View File
@@ -44,6 +44,11 @@ def clean_price(v):
return float(m.group()) if m else None return float(m.group()) if m else None
def price_from_meta(h): def price_from_meta(h):
# Shopware/Microdata: itemprop="price" content="..."
mi=re.search(r'itemprop=["\']price["\'][^>]*content=["\']([^"\']+)', h, re.I) or re.search(r'content=["\']([^"\']+)["\'][^>]*itemprop=["\']price["\']', h, re.I)
if mi:
p=clean_price(mi.group(1));
if p: return p
for pat in ("product:price:amount","og:price:amount"): for pat in ("product:price:amount","og:price:amount"):
m=re.search(r'<meta[^>]+(?:property|name)=["\']'+pat+r'["\'][^>]+content=["\']([^"\']+)', h, re.I) or \ m=re.search(r'<meta[^>]+(?:property|name)=["\']'+pat+r'["\'][^>]+content=["\']([^"\']+)', h, re.I) or \
re.search(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+(?:property|name)=["\']'+pat+r'["\']', h, re.I) re.search(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+(?:property|name)=["\']'+pat+r'["\']', h, re.I)
@@ -74,9 +79,9 @@ def parse(htmltext):
online = ("in den warenkorb" in low) or ("lieferung" in low and "nicht möglich" not in low) or (avail and "instock" in avail.lower()) online = ("in den warenkorb" in low) or ("lieferung" in low and "nicht möglich" not in low) or (avail and "instock" in avail.lower())
if price is None: if price is None:
price=price_from_meta(htmltext) price=price_from_meta(htmltext)
if price is None: # Plausibilitäts-Schranke: Klimageräte kosten 3-stellig; verwirf Ausreißer (Raten/Versand)
mp=_num(htmltext[:200000]) if price is not None and price < 150:
price=mp price=None
if in_store: availability="in_stock" if in_store: availability="in_stock"
elif online: availability="online_available" elif online: availability="online_available"
elif price is not None: availability="out_of_stock" elif price is not None: availability="out_of_stock"