From 6b127737acc4c61c8aaa0d12b28508384a9a755c Mon Sep 17 00:00:00 2001 From: "Till (Cowork)" Date: Mon, 29 Jun 2026 12:29:14 +0000 Subject: [PATCH] Crawler: Auto-Discovery der Klimageraete-Kategorieseiten (toom+OBI) findet Produkte selbst (Daten-Hygiene-Filter, Zubehoer raus); 3-Min-Takt; paralleles Crawling; discovered-Flag + Counts im Snapshot --- .../__pycache__/discovery.cpython-310.pyc | Bin 0 -> 2559 bytes livecrawler/app.py | 25 ++++++--- livecrawler/discovery.py | 50 ++++++++++++++++++ 3 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 livecrawler/__pycache__/discovery.cpython-310.pyc create mode 100644 livecrawler/discovery.py diff --git a/livecrawler/__pycache__/discovery.cpython-310.pyc b/livecrawler/__pycache__/discovery.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a87743d0a85b65df351a3cbc3fa75b89bec9b27 GIT binary patch literal 2559 zcmZWqPj4H?72jDd$tCs2vKuQ^>IQo;Dn_AbAOVWHsar`FYE_0Ugbso#3WV96A-Upm zcRe#J+Qbk@kQBW%hn|xHB;Z5)9dgs9pJA>!=_?pT+utmuAWcc^n|W{M&AdOq_a<9e zX*&4T`R^sZ+@Ml+lhfkh?Fz47|gt-YhCal6dR%JC-XFhANCR_T(XUnB*g{`t1 zkTk5U?;g9H%qnb+lQVDb9#>25<`q|Cx7bgFz(??LG@OrJ_m&11i)o!$m^jtdBl?EL+<_Hw(rW zucE<~2|FR<(XdGSO$swdRKdL6~eYm&Z4Z?)SA@=ny zz7;Sz4faGPWxG4b^7fJ7v4qim?1598i;)Mpjaj4)Sa9)eh{)K@2+bwH;XE;(^~d-j z!c7rv<38M<+!!d8%ln;K@*XOz%7hWyqGEZWqOWH0Mbq)~Gg?I!TkjLL%t%R!3Ibc%6X{MuW z)Z<)_V#cXvJjv4jAl9--6_0Q<6;c5{TJX^@(@JDkOnG0284%l|8wl6tIiMORRAoZ< zcnSqULx$WdO~a(Ze z4+}m>0NN0l){7Ir=j;-&Y~iF&Eb{H_B*c6eD~zKoRe+bKjONyxYj<0b#nSc9LaL5$ z51-rxQO+5$9Z@?o7ah@^yO5wWe&tPxa%Kb_($!>MnR#<(R-HRjq6u?Y?N9E8lLy*C zr>j4i{Q4x^7x1ubVVV@$_rv$gRr{U|OptmNA%$EJ9NkJWpuE4LLAXsYINcs)|*@TdwhJ0T?3x;fS9>;q@oL zw3Km{$_{$TId4Z<+5@yw)kZsI95pyfRp;=k1EUVx&$2A(l;JOXtLP;0IY$E?Bc&a7 z-M_cG#3w-izoT;b1GnZ9|6>;Zd=Xq6CzHp{^tEM&O&8pC-Z#dXd zo4Yva<7r*jXLU&Z4QEcK{uS3KxrXv*4eif~c&M6Jv;F7qV0RN|K#$etDCirLw30V@(hWs+XpBs zlv(R#Ns^6E1}=H%``!x+ap7Ci0GU*62HWB$=B@gV7ER;( zC&73&wWd;t#5Avy&Y7rMy){$c+j+M0`04LWrO%aFvhQpmPos=MMRK(Ot8ejxH(cN? z;V+b__C;3Y#yi~KKQgsNuQgux@ZiW)C}XBZ^Bj+=slq)jN*tNS{)>p`3IJ{`nOcMw zG*!0U=cZzVK_WwNyabL>Ga0JLkdd%4GxZ!VKw6B%U2tVFEXW*Nl31N^O0YWa0Y{eSxvAM5Uayw8^1a&*N5sJDoC+YSv=KRsnxbb7i- z#xEc=b@cQc4PUH-6Wf;Dv&2?Psr>`VE auch bei vielen Quellen schnell genug für 5-Min-Takt - results=await asyncio.gather(*[guarded(s) for s in SOURCES]) - results=list(results) + # parallel (begrenzt) -> auch bei vielen Quellen schnell genug für kurzen Takt + results=list(await asyncio.gather(*[guarded(s) for s in sources])) # Restock-Events prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) } hist=load(HIST,{"events":[]}) @@ -143,6 +155,7 @@ async def crawl(): gone=[o for o in results if o.get("availability")=="discontinued"] blocked=[o for o in results if o.get("availability")=="blocked"] snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),"blocked":len(blocked),"proxy":bool(PROXY), + "curatedCount":len(SOURCES),"discoveredCount":discovered,"totalSources":len(sources), "offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]} json.dump(snap, open(LIVE,"w"), ensure_ascii=False) json.dump(hist, open(HIST,"w"), ensure_ascii=False) diff --git a/livecrawler/discovery.py b/livecrawler/discovery.py new file mode 100644 index 0000000..83d9418 --- /dev/null +++ b/livecrawler/discovery.py @@ -0,0 +1,50 @@ +"""Auto-Discovery: liest die Klimageräte-Kategorieseiten der scrapebaren Baumärkte +und findet Produktseiten SELBST (statt hartkodierter URLs). So wächst die Abdeckung +automatisch mit dem Sortiment. Server-gerenderte Chains: toom, OBI. +Hornbach-Listing = JS (liefert keine Links), Bauhaus = IP-Block (403) -> brauchen Proxy/Headless. +Curated SOURCES bleiben für den Cross-Shop-Preisvergleich (gleiche productId über Shops).""" +import re + +# Kategorie-Seiten + Produktlink-Muster je Chain +CATALOG = [ + {"chain":"toom","kind":"baumarkt","cc":"DE","base":"https://toom.de", + "cat":"https://toom.de/c/bauen-renovieren/klima-lueftung/klimaanlagen/klimageraete", + "re":r'/p/([a-z0-9-]+)/(\d+)', "slug_grp":1, "id_grp":2}, + {"chain":"OBI","kind":"baumarkt","cc":"DE","base":"https://www.obi.de", + "cat":"https://www.obi.de/search/klimager%C3%A4te/", + "re":r'/p/(\d+)/([a-z0-9-]+)', "slug_grp":2, "id_grp":1}, +] +# Nur echte Klimageräte behalten, Zubehör/Empfehlungs-Querlinks raus (Daten-Hygiene) +KEEP = re.compile(r'(klimage|klimaanlage|klimasplit|split-klima|portasplit|mobiles?-klima|btu|pinguino|comfee|midea|delonghi|suntec|klarstein|remko|trotec|aeg|bosch)', re.I) +SKIP = re.compile(r'(ventilator|fensterabdicht|abluftschlauch|fernbedien|ersatzfilter|halterung|wandhalter|zubeh|konvektor|heizk|luftentfeucht|luftk\w+hler|tischvent|standvent|deckenvent|abdeckhaube|abdeckung|reinig|kuehlverbind|k\w+hlverbind|kuehlleitung|\bleitung\b|\bkit\b|montage|adapter|schlauch)', re.I) + +def _name(slug): + s=re.sub(r'\b(eek|a|aplus|aplusplus|schwarz|weiss|grau|inkl|mit|und)\b',' ',slug.replace('-',' '),flags=re.I) + s=re.sub(r'\s+',' ',s).strip().title() + return s[:60] or slug + +async def discover(client, cap=20): + out=[] + for c in CATALOG: + try: + r=await client.get(c["cat"], timeout=25, follow_redirects=True) + if r.status_code!=200: + continue + seen=set(); found=0 + for m in re.finditer(c["re"], r.text): + slug=m.group(c["slug_grp"]); pid_num=m.group(c["id_grp"]) + if not KEEP.search(slug) or SKIP.search(slug): + continue + url=c["base"]+m.group(0) + if url in seen: + continue + seen.add(url) + out.append({"chain":c["chain"],"kind":c["kind"],"cc":c["cc"], + "productId":f"disc-{c['chain'].lower()}-{pid_num}", + "name":_name(slug),"url":url,"discovered":True}) + found+=1 + if found>=cap: + break + except Exception: + pass + return out