scripts/scrape.js
#!/usr/bin/env node
// scrape.js — thin Node wrapper: shared feed core + local file storage.
// Fetches every registered source at offered depth, dedupes content-addressed
// across all days, appends to raw/<date>/articles.jsonl, writes source health.
import fs from "node:fs";
import path from "node:path";
import { DATA_ROOT, loadSources, parseArgs, readAllJsonl, readJsonl, utcDate } from "./lib.js";
import { fetchSource, parseFeed, articleFromItem } from "../pipeline/feed.js";
const { args } = parseArgs(process.argv.slice(2), { date: { takes: "value", default: null } });
const date = args.date || utcDate();
const rawDir = path.join(DATA_ROOT, "raw", date);
const articlesFile = path.join(rawDir, "articles.jsonl");
const sourcesFile = path.join(rawDir, "sources.json");
const seen = new Set(readAllJsonl(path.join(DATA_ROOT, "raw"), "articles.jsonl").map((a) => a.id));
const sourceStatus = fs.existsSync(sourcesFile) ? JSON.parse(fs.readFileSync(sourcesFile, "utf8")) : {};
const registry = Object.fromEntries(loadSources().map((s) => [s.id, s]));
const meta = (src) => ({ name: src.name, site: src.site || null, takes: src.takes || null, basis: src.basis || null, depth: src.depth });
let totalNew = 0;
const report = [];
for (const src of Object.values(registry)) {
if (src.via === "deferred") {
sourceStatus[src.id] = { status: "deferred", http: null, fetched: 0, new: 0, ...meta(src) };
report.push(`${src.id}: deferred (${(src.note || src.basis || "no route").slice(0, 80)})`);
continue;
}
try {
const { xml, http } = await fetchSource(src, fetch);
const items = parseFeed(xml);
let fresh = 0;
for (const it of items) {
const article = articleFromItem(src, it, seen, date);
if (!article) continue;
fs.mkdirSync(rawDir, { recursive: true });
fs.appendFileSync(articlesFile, JSON.stringify(article) + "\n", "utf8");
fresh++;
}
sourceStatus[src.id] = { status: "ok", http, fetched: items.length, new: fresh, ...meta(src) };
totalNew += fresh;
report.push(`${src.id}: ${items.length} fetched, ${fresh} new`);
} catch (e) {
sourceStatus[src.id] = { status: "failed", http: null, error: String(e.message), fetched: 0, new: 0, ...meta(src) };
report.push(`${src.id}: FAILED (${e.message})`);
}
}
fs.mkdirSync(rawDir, { recursive: true });
fs.writeFileSync(sourcesFile, JSON.stringify(sourceStatus, null, 2) + "\n");
console.log(`scrape ${date}: ${totalNew} new articles -> raw/${date}/articles.jsonl (${fs.existsSync(articlesFile) ? readJsonl(articlesFile).length : 0} today)`);
for (const r of report) console.log(` ${r}`);