scripts/distill.js
#!/usr/bin/env node
// distill.js — thin Node wrapper: shared distillation core + REST AI client.
// Budget-guarded: stops at the daily cap. Failures are recorded, never retried
// silently; a deliberate retry runs only with --retry.
import fs from "node:fs";
import path from "node:path";
import {
budgetCheck, DATA_ROOT, loadConfig, logSpend, nowIso, parseArgs, PROMPT_VERSION,
readAllJsonl, runModel, loadTopics, spendToday,
} from "./lib.js";
import { DISTILL_PROMPT } from "../pipeline/prompts.js";
import { distillUserMessage, topicListForPrompt, essenceFromResponse } from "../pipeline/distill-core.js";
const { args } = parseArgs(process.argv.slice(2), {
date: { takes: "value", default: null },
limit: { takes: "value", default: null },
retry: { takes: "none", default: false },
});
const config = loadConfig();
const date = args.date || new Date().toISOString().slice(0, 10);
const articles = readAllJsonl(path.join(DATA_ROOT, "raw"), "articles.jsonl");
const essenceBy = {};
for (const e of readAllJsonl(path.join(DATA_ROOT, "distilled"), "essence.jsonl")) {
const prev = essenceBy[e.article.raw_id];
if (!prev || (prev.failure && !e.failure)) essenceBy[e.article.raw_id] = e;
}
const isPending = (a) => {
const e = essenceBy[a.id];
if (!e) return true;
return args.retry && !!e.failure;
};
const pending = articles.filter(isPending);
const cap = args.limit ? parseInt(args.limit, 10) : config.max_articles_per_run;
const queue = pending.slice(0, cap);
if (!queue.length) {
console.log(`distill ${date}: nothing pending (${articles.length} articles, ${Object.keys(essenceBy).length} distilled).`);
process.exit(0);
}
if (!budgetCheck(config, "distill").ok) process.exit(0);
console.log(`distill ${date}: ${queue.length} pending (${pending.length} total), $${(config.daily_budget_usd - spendToday()).toFixed(3)} budget left`);
const topics = loadTopics();
const topicText = topicListForPrompt(topics);
const outFile = path.join(DATA_ROOT, "distilled", date, "essence.jsonl");
let spent = 0, ok = 0, failed = 0;
for (const a of queue) {
try {
const r = await runModel(config.distill_model, [
{ role: "system", content: DISTILL_PROMPT },
{ role: "user", content: distillUserMessage(a, topicText) },
], { temperature: 0.2, max_tokens: 3000 });
logSpend({ script: "distill", model: config.distill_model, tokensIn: r.tokensIn, tokensOut: r.tokensOut, cost: r.cost });
if (r.cost != null) spent += r.cost;
const out = essenceFromResponse(r.text, a, {
model: config.distill_model, promptVersion: PROMPT_VERSION,
tokensIn: r.tokensIn, tokensOut: r.tokensOut, cost: r.cost, at: nowIso(),
});
out.ok ? ok++ : failed++;
if (!out.ok) console.error(` parse failure on ${a.id}: ${out.error}`);
fs.mkdirSync(path.dirname(outFile), { recursive: true });
fs.appendFileSync(outFile, JSON.stringify(out.record) + "\n", "utf8");
} catch (apiErr) {
console.error(` API error on ${a.id}: ${apiErr.message} — stopping this run`);
break;
}
if (config.daily_budget_usd - spendToday() <= 0) {
console.log(` [budget] daily cap reached, ${queue.length - ok - failed} deferred`);
break;
}
}
console.log(`distill ${date}: ${ok} essences, ${failed} parse failures, ~$${spent.toFixed(4)} this run (day total $${spendToday().toFixed(4)})`);