122 lines
6.0 KiB
Python
122 lines
6.0 KiB
Python
"""Kühlfinder Live-Crawler — eigener Verfügbarkeits-/Preis-Scraper.
|
|
Muster gespiegelt vom AIDA Preisradar (FastAPI + httpx + APScheduler + /app/data + Coolify).
|
|
Scrapt echte, server-gerenderte Händlerseiten (OBI, Bauhaus), persistiert Snapshots +
|
|
Restock-Historie, liefert /api/live (CORS). Keine Drittdaten (kein braucheklima-Import)."""
|
|
import os, re, json, html, logging, asyncio
|
|
from datetime import datetime, timezone
|
|
import httpx
|
|
from fastapi import FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.middleware.gzip import GZipMiddleware
|
|
from fastapi.responses import JSONResponse
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.interval import IntervalTrigger
|
|
from sources import SOURCES
|
|
|
|
logging.basicConfig(level=logging.INFO); log = logging.getLogger("kf-crawler")
|
|
DATA = os.getenv("DATA_DIR","/app/data"); os.makedirs(DATA, exist_ok=True)
|
|
LIVE = os.path.join(DATA,"live.json"); HIST = os.path.join(DATA,"history.json")
|
|
INTERVAL_MIN = int(os.getenv("CRAWL_INTERVAL_MIN","30"))
|
|
UA = {"User-Agent":"Mozilla/5.0 (compatible; KuehlfinderBot/0.1; +https://kuehlfinder.de/bot)","Accept-Language":"de-DE"}
|
|
|
|
def _num(s):
|
|
s=s.replace(".","").replace("\xa0"," ")
|
|
m=re.search(r"(\d{2,4})[,](\d{2})",s) or re.search(r"(\d{2,4}),-",s) or re.search(r"(\d{2,4})\s*€",s)
|
|
if not m: return None
|
|
g=m.groups()
|
|
return float(g[0]) + (float(g[1])/100 if len(g)>1 and g[1] else 0)
|
|
|
|
def parse(htmltext):
|
|
# 1) JSON-LD Offer
|
|
price=None; avail=None
|
|
for m in re.finditer(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', htmltext, re.S|re.I):
|
|
try: data=json.loads(m.group(1))
|
|
except Exception: continue
|
|
for node in (data if isinstance(data,list) else [data]):
|
|
graph = node.get("@graph",[node]) if isinstance(node,dict) else [node]
|
|
for g in graph:
|
|
if not isinstance(g,dict): continue
|
|
off=g.get("offers")
|
|
if off:
|
|
off=off[0] if isinstance(off,list) else off
|
|
try: price=float(off.get("price") or off.get("lowPrice"))
|
|
except Exception: pass
|
|
avail=str(off.get("availability") or "")
|
|
# 2) Heuristik
|
|
low=htmltext.lower()
|
|
in_store = ("vorrätig" in low or "verfügbar" in low) and not ("nicht vorrätig" in low or "nicht verfügbar" in low)
|
|
online = ("in den warenkorb" in low) or ("lieferung" in low and "nicht möglich" not in low) or (avail and "instock" in avail.lower())
|
|
if price is None:
|
|
mp=_num(htmltext[:200000])
|
|
price=mp
|
|
availability = "in_stock" if in_store else ("online_available" if online else "out_of_stock")
|
|
return price, availability
|
|
|
|
async def fetch_one(client, src):
|
|
try:
|
|
r= await client.get(src["url"], headers=UA, timeout=20, follow_redirects=True)
|
|
if r.status_code in (404,410):
|
|
# Produktseite verschwunden -> als entfernt kennzeichnen (das kann braucheklima nicht)
|
|
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
|
|
"url":src["url"],"price":None,"availability":"discontinued","status":r.status_code}
|
|
if r.status_code!=200:
|
|
return {**src,"price":None,"availability":"unknown","status":r.status_code}
|
|
price, avail = parse(r.text)
|
|
return {"chain":src["chain"],"kind":src["kind"],"productId":src["productId"],"name":src["name"],
|
|
"url":src["url"],"price":price,"availability":avail}
|
|
except Exception as e:
|
|
log.warning("fetch fail %s: %s", src["url"], e)
|
|
return {**src,"price":None,"availability":"unknown"}
|
|
|
|
def load(path,default):
|
|
try:
|
|
with open(path) as f: return json.load(f)
|
|
except Exception: return default
|
|
|
|
async def crawl():
|
|
now=datetime.now(timezone.utc).isoformat()
|
|
async with httpx.AsyncClient() as client:
|
|
results=[]
|
|
for src in SOURCES:
|
|
results.append(await fetch_one(client, src))
|
|
await asyncio.sleep(1.2) # höflich: Jitter/Rate-Limit
|
|
# Restock-Events
|
|
prev={ (o["chain"],o["productId"]): o for o in load(LIVE,{}).get("offers",[]) }
|
|
hist=load(HIST,{"events":[]})
|
|
for o in results:
|
|
p=prev.get((o["chain"],o["productId"]))
|
|
if p and p.get("availability")=="out_of_stock" and o.get("availability") in ("in_stock","online_available"):
|
|
hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now})
|
|
if p and p.get("availability")!="discontinued" and o.get("availability")=="discontinued":
|
|
hist["events"].append({"chain":o["chain"],"productId":o["productId"],"name":o["name"],"at":now,"type":"discontinued"})
|
|
hist["events"]=hist["events"][-200:]
|
|
# Preis-Aggregat je Produkt
|
|
byp={}
|
|
for o in results:
|
|
if o.get("price"): byp.setdefault(o["productId"],[]).append(o["price"])
|
|
agg={k:{"min":min(v),"max":max(v),"n":len(v)} for k,v in byp.items()}
|
|
gone=[o for o in results if o.get("availability")=="discontinued"]
|
|
snap={"generatedAt":now,"source":"kuehlfinder-livecrawler","intervalMin":INTERVAL_MIN,"discontinued":len(gone),
|
|
"offers":results,"priceByProduct":agg,"feed":hist["events"][-20:][::-1]}
|
|
json.dump(snap, open(LIVE,"w"), ensure_ascii=False)
|
|
json.dump(hist, open(HIST,"w"), ensure_ascii=False)
|
|
log.info("crawl done: %d offers, %d events", len(results), len(hist["events"]))
|
|
return snap
|
|
|
|
app=FastAPI(title="Kühlfinder Live-Crawler", version="0.1.0")
|
|
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"])
|
|
app.add_middleware(GZipMiddleware, minimum_size=400)
|
|
sched=AsyncIOScheduler()
|
|
|
|
@app.on_event("startup")
|
|
async def _start():
|
|
await crawl()
|
|
sched.add_job(crawl, IntervalTrigger(minutes=INTERVAL_MIN), id="crawl", max_instances=1, coalesce=True)
|
|
sched.start()
|
|
|
|
@app.get("/health")
|
|
def health(): return {"ok":True}
|
|
|
|
@app.get("/api/live")
|
|
def live(): return JSONResponse(load(LIVE,{"offers":[],"note":"warming up"}))
|