pipeline/distill-core.js
// Distillation core — message construction and response parsing, shared by both
// runtimes. The AI call itself is injected by the caller (Node REST client or
// the Worker's AI binding), so this module has no transport.
import { hashId } from "./util.js";
import { extractJson } from "./util.js";
export function distillUserMessage(article, topicListText) {
return [
`Source: ${article.source_name} (${article.summary ? "headline and summary from the newsroom's feed" : "headline only from the newsroom's feed"})`,
`Title: ${article.title}`,
article.summary ? `Summary: ${article.summary}` : "Summary: none",
"Topic ids you may use (topic: subtopics):",
topicListText,
].join("\n");
}
export function topicListForPrompt(topics) {
return topics
.filter((t) => t.id !== "other")
.map((t) => `${t.id}: ${t.subtopics.join(", ")}`)
.join("\n");
}
function valid(e) {
return typeof e.essence === "string" && e.essence.length > 0 &&
Array.isArray(e.topics) && e.topics.length > 0 &&
typeof e.event_type === "string" && e.event_type.length > 0;
}
// -> { record } : a conforming essence record, or a recorded parse failure.
// Failures are data, never retried silently.
export function essenceFromResponse(text, article, { model, promptVersion, tokensIn, tokensOut, cost, at }) {
const meta = { model, prompt_version: promptVersion, distilled_at: at, tokens_in: tokensIn ?? null, tokens_out: tokensOut ?? null, cost_usd: cost ?? null };
try {
const e = extractJson(text);
if (!valid(e)) throw new Error("shape validation failed");
return {
ok: true,
record: {
id: hashId("ess", article.id),
status: "unverified",
lane: "net",
article: { raw_id: article.id, source_id: article.source_id, source_name: article.source_name, url: article.url, title: article.title, date: article.date },
essence: String(e.essence).trim(),
principles: Array.isArray(e.principles) ? e.principles.map(String) : [],
topics: e.topics.map((t) => ({ topic: String(t.topic || "other"), subtopic: String(t.subtopic || "other") })),
actors: Array.isArray(e.actors) ? e.actors.map(String) : [],
locations: Array.isArray(e.locations) ? e.locations.filter((l) => /^[A-Z]{2}$/.test(String(l.country_code || ""))).map((l) => ({ name: String(l.name), country_code: String(l.country_code).toUpperCase() })) : [],
event_type: String(e.event_type),
uncertainty: e.uncertainty ? String(e.uncertainty) : "",
distillation: meta,
failure: null,
},
};
} catch (ve) {
return {
ok: false,
error: String(ve.message || ve),
record: {
id: hashId("essf", article.id), // failure ids never collide with real essences
status: "unverified",
lane: "net",
article: { raw_id: article.id, source_id: article.source_id, source_name: article.source_name, url: article.url, title: article.title, date: article.date },
essence: "(distillation parse failure — recorded, not retried)",
principles: [], topics: [{ topic: "other", subtopic: "other" }], actors: [], locations: [],
event_type: "report", uncertainty: "",
distillation: meta,
failure: { kind: "parse_failure", raw: String(text).slice(0, 500) },
},
};
}
}