till/kuehlfinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v5: Mollie (EU) statt Stripe, Sitemap + llms-full.txt, /prognose-Methodik, ausgebauter Datenschutz/Transparenz, Scraper-Adapter toom/bauhaus/hornbach + compose/cron-ready, checkedAt-Stempel im Monitor

Browse Source
This commit is contained in:
Till Heidrich
2026-06-28 07:26:21 +00:00
parent 7636a00d9f
commit a266b14482
18 changed files with 265 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scraper/Dockerfile.scraper
+7
View File
@@ -0,0 +1,7 @@
FROM node:22-alpine
WORKDIR /app
COPY scraper/ ./scraper/
COPY public/data ./public/data
WORKDIR /app/scraper
# Coolify Scheduled Task: "node run.mjs" (z.B. */30 * * * *)
CMD ["node","run.mjs"]
scraper/adapters/bauhaus.mjs
+18
View File
@@ -0,0 +1,18 @@
// Bauhaus-Adapter (Muster wie OBI: Produktseite → Preis via JSON-LD + Markt-Status-Heuristik).
// TODO: products{} mit echten Produkt-URLs/Artikelnummern füllen; Per-Markt-Endpoint ergänzen.
const UA = 'KuehlfinderBot/0.1 (+https://kuehlfinder.de/bot)';
export const meta = { id: 'bauhaus', name: 'Bauhaus', kind: 'baumarkt' };
export const products = { /* 'portasplit': 'https://www....' */ };
function jsonld(html){const o=[];const re=/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi;let m;while((m=re.exec(html))){try{o.push(JSON.parse(m[1]));}catch{}}return o;}
function offer(ld){for(const n of (Array.isArray(ld)?ld:[ld])){const g=n['@graph']||[n];for(const x of g){if(x&&(x['@type']==='Product'||x.offers)){const of=Array.isArray(x.offers)?x.offers[0]:x.offers;if(of)return{price:Number(of.price??of.lowPrice),availability:String(of.availability||'')};}}}return null;}
export async function fetchProduct(productId,{marketId}={}){
const url=products[productId]; if(!url) return null;
const res=await fetch(url,{headers:{'User-Agent':UA,'Accept-Language':'de-DE'}}); if(!res.ok) throw new Error('Bauhaus '+res.status);
const html=await res.text(); const o=jsonld(html).map(offer).find(Boolean);
const inStore=/vorrätig|verfügbar/i.test(html)&&!/nicht (vorrätig|verfügbar)/i.test(html);
const price=o?.price ?? Number((html.match(/(\d{2,4})[,.](\d{2})\s*€/)||[])[0]?.replace(/[^\d,]/g,'').replace(',','.'));
return { productId, source:'bauhaus', marketId:marketId??null, price: price||null,
availability: inStore?'in_stock':(o&&/InStock/i.test(o.availability)?'online_available':'out_of_stock'),
checkedAt:new Date().toISOString(), url };
}
export async function fetchStores(){ return []; }
scraper/adapters/hornbach.mjs
+18
View File
@@ -0,0 +1,18 @@
// Hornbach-Adapter (Muster wie OBI: Produktseite → Preis via JSON-LD + Markt-Status-Heuristik).
// TODO: products{} mit echten Produkt-URLs/Artikelnummern füllen; Per-Markt-Endpoint ergänzen.
const UA = 'KuehlfinderBot/0.1 (+https://kuehlfinder.de/bot)';
export const meta = { id: 'hornbach', name: 'Hornbach', kind: 'baumarkt' };
export const products = { /* 'portasplit': 'https://www....' */ };
function jsonld(html){const o=[];const re=/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi;let m;while((m=re.exec(html))){try{o.push(JSON.parse(m[1]));}catch{}}return o;}
function offer(ld){for(const n of (Array.isArray(ld)?ld:[ld])){const g=n['@graph']||[n];for(const x of g){if(x&&(x['@type']==='Product'||x.offers)){const of=Array.isArray(x.offers)?x.offers[0]:x.offers;if(of)return{price:Number(of.price??of.lowPrice),availability:String(of.availability||'')};}}}return null;}
export async function fetchProduct(productId,{marketId}={}){
const url=products[productId]; if(!url) return null;
const res=await fetch(url,{headers:{'User-Agent':UA,'Accept-Language':'de-DE'}}); if(!res.ok) throw new Error('Hornbach '+res.status);
const html=await res.text(); const o=jsonld(html).map(offer).find(Boolean);
const inStore=/vorrätig|verfügbar/i.test(html)&&!/nicht (vorrätig|verfügbar)/i.test(html);
const price=o?.price ?? Number((html.match(/(\d{2,4})[,.](\d{2})\s*€/)||[])[0]?.replace(/[^\d,]/g,'').replace(',','.'));
return { productId, source:'hornbach', marketId:marketId??null, price: price||null,
availability: inStore?'in_stock':(o&&/InStock/i.test(o.availability)?'online_available':'out_of_stock'),
checkedAt:new Date().toISOString(), url };
}
export async function fetchStores(){ return []; }
scraper/adapters/toom.mjs
+18
View File
@@ -0,0 +1,18 @@
// Toom-Adapter (Muster wie OBI: Produktseite → Preis via JSON-LD + Markt-Status-Heuristik).
// TODO: products{} mit echten Produkt-URLs/Artikelnummern füllen; Per-Markt-Endpoint ergänzen.
const UA = 'KuehlfinderBot/0.1 (+https://kuehlfinder.de/bot)';
export const meta = { id: 'toom', name: 'Toom', kind: 'baumarkt' };
export const products = { /* 'portasplit': 'https://www....' */ };
function jsonld(html){const o=[];const re=/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi;let m;while((m=re.exec(html))){try{o.push(JSON.parse(m[1]));}catch{}}return o;}
function offer(ld){for(const n of (Array.isArray(ld)?ld:[ld])){const g=n['@graph']||[n];for(const x of g){if(x&&(x['@type']==='Product'||x.offers)){const of=Array.isArray(x.offers)?x.offers[0]:x.offers;if(of)return{price:Number(of.price??of.lowPrice),availability:String(of.availability||'')};}}}return null;}
export async function fetchProduct(productId,{marketId}={}){
const url=products[productId]; if(!url) return null;
const res=await fetch(url,{headers:{'User-Agent':UA,'Accept-Language':'de-DE'}}); if(!res.ok) throw new Error('Toom '+res.status);
const html=await res.text(); const o=jsonld(html).map(offer).find(Boolean);
const inStore=/vorrätig|verfügbar/i.test(html)&&!/nicht (vorrätig|verfügbar)/i.test(html);
const price=o?.price ?? Number((html.match(/(\d{2,4})[,.](\d{2})\s*€/)||[])[0]?.replace(/[^\d,]/g,'').replace(',','.'));
return { productId, source:'toom', marketId:marketId??null, price: price||null,
availability: inStore?'in_stock':(o&&/InStock/i.test(o.availability)?'online_available':'out_of_stock'),
checkedAt:new Date().toISOString(), url };
}
export async function fetchStores(){ return []; }
scraper/docker-compose.yml
+19
View File
@@ -0,0 +1,19 @@
# Cron-ready Setup: Scraper schreibt availability/history in ein gemeinsames Volume,
# das der nginx-Webcontainer unter /data ausliefert → Seite wird live ohne Rebuild.
# In Coolify als "Docker Compose"-Resource deploybar (ersetzt später den reinen static-Deploy).
services:
web:
build: { context: .., dockerfile: Dockerfile }
volumes:
- livedata:/usr/share/nginx/html/data
ports: ["80"]
scraper:
build: { context: ., dockerfile: Dockerfile.scraper }
environment:
- TZ=Europe/Berlin
volumes:
- livedata:/app/public/data
# Cron via Coolify Scheduled Task: command = node run.mjs (alle 30 Min)
restart: unless-stopped
volumes:
livedata:
scraper/run.mjs
+4 -1
View File
@@ -3,8 +3,11 @@
// hängt Restock-Events an history.json (Basis für die Prognose).
import { readFile, writeFile } from 'node:fs/promises';
import * as obi from './adapters/obi.mjs';
import * as toom from './adapters/toom.mjs';
import * as bauhaus from './adapters/bauhaus.mjs';
import * as hornbach from './adapters/hornbach.mjs';
const ADAPTERS = [obi /*, toom, bauhaus, hornbach, mediamarkt, saturn, expert, euronics, amazon */];
const ADAPTERS = [obi, toom, bauhaus, hornbach /*, mediamarkt, saturn, expert, euronics, amazon */];
const OUT = new URL('../public/data/', import.meta.url);
async function loadJson(name, fallback) {