scripts/pipeline.js
#!/usr/bin/env node
// pipeline.js — one full cycle of the unverified lane, in order:
// scrape -> distill -> aggregate -> gis. With --commit: stage the data repo,
// commit "Cycle <date>: ...", and push to the the self-hosted remote bare remote (best effort).
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { DATA_ROOT, parseArgs, utcDate } from "./lib.js";
const { args } = parseArgs(process.argv.slice(2), {
commit: { takes: "none", default: false },
});
const steps = ["scrape.js", "distill.js", "cluster.js", "aggregate.js", "gis.js"];
for (const s of steps) {
console.log(`\n=== ${s} ===`);
const r = spawnSync(process.execPath, [path.join(import.meta.dirname, s)], { stdio: "inherit" });
if (r.status !== 0) {
console.error(`pipeline: ${s} exited ${r.status} — continuing is unsafe, stopping cycle (stages are idempotent; re-run anytime)`);
process.exit(1);
}
}
if (!args.commit) process.exit(0);
const date = utcDate();
const git = (cmd, opts = {}) => spawnSync("git", ["-C", DATA_ROOT, ...cmd.split(" ")], { stdio: "pipe", encoding: "utf8", ...opts });
let msg = `Cycle ${date}`;
try {
const day = JSON.parse(fs.readFileSync(path.join(DATA_ROOT, "aggregates", date, "day.json"), "utf8"));
msg = `Cycle ${date}: ${day.counts.articles} articles, ${day.counts.essences} essences`;
} catch {}
const add = git("add -A");
const commit = spawnSync("git", ["-C", DATA_ROOT, "commit", "-m", msg], { stdio: "pipe", encoding: "utf8" });
if (commit.status !== 0 && !/nothing to commit/.test(String(commit.stdout))) {
console.error(`pipeline: data commit failed: ${commit.stderr}`);
process.exit(1);
}
console.log(commit.stdout.trim() || "data repo: nothing to commit");
const push = git("push origin main");
if (push.status === 0) console.log("data repo: pushed to origin");
else console.warn(`data repo: push failed (${String(push.stderr).trim().split("\n")[0]}) — push manually when the self-hosted remote is reachable`);