pipeline/util.js
// Shared utilities for the emergence pipeline — runtime-agnostic by design:
// no Node built-ins, no DOM, no fetch assumptions. Both the local scripts
// (scripts/*.js) and the Cloudflare Worker (worker/index.js) import from here.
import crypto from "node:crypto";
export const PRICES = {
"@cf/zai-org/glm-5.3-flash": { in: 0.15, out: 0.5 },
"@cf/zai-org/glm-5.3": { in: 1.4, out: 4.4 },
};
export const PROMPT_VERSION = "1.1";
export const utcDate = (d = Date.now()) => new Date(d).toISOString().slice(0, 10);
export const prevDate = (date) => new Date(Date.parse(date + "T00:00:00Z") - 86400000).toISOString().slice(0, 10);
export const nowIso = () => new Date().toISOString();
export function hashId(prefix, content) {
const h = crypto.createHash("sha256").update(content).digest("hex").slice(0, 12);
return `${prefix}_${h}`;
}
// Controlled YAML subset used by sources/sources.yaml and ontology/topics.yaml.
export function parseRegistryYaml(text) {
const items = [];
let cur = null;
for (const raw of String(text).split("\n")) {
const line = raw.replace(/\r$/, "");
if (!line.trim() || /^\s*#/.test(line)) continue;
const indent = line.length - line.trimStart().length;
const t = line.trim();
if (indent === 0 && t.startsWith("- ")) {
cur = {};
items.push(cur);
const rest = t.slice(2).trim();
const ci = rest.indexOf(":");
if (ci > -1) cur[rest.slice(0, ci).trim()] = rest.slice(ci + 1).trim();
continue;
}
const ci = t.indexOf(":");
if (ci === -1 || indent !== 2) continue;
const key = t.slice(0, ci).trim();
const val = t.slice(ci + 1).trim();
if (val === "") continue;
cur[key] = key === "subtopics" ? val.split(",").map((s) => s.trim()).filter(Boolean) : val;
}
return items;
}
/* ---------- text similarity (cluster phase A) ---------- */
export const STOPWORDS = new Set("the a an and or but if while of to in on for with by at from as is are was were be been being it its this that these those has have had had not no nor so than then there here what which who whom when where why how will would can could may might must do does did done about over under after before between during into through against without within across per via says say said report reports according new news latest more most also their they them his her his its".split(" "));
export function words(s) {
const set = new Set();
for (const w of String(s).toLowerCase().match(/[a-z][a-z'-]{2,}/g) || []) {
const t = w.replace(/['-]/g, "");
if (!STOPWORDS.has(w) && !STOPWORDS.has(t)) set.add(t);
}
return set;
}
export function jaccard(a, b) {
if (!a.size || !b.size) return 0;
let inter = 0;
for (const w of a) if (b.has(w)) inter++;
return inter / (a.size + b.size - inter);
}
export function extractJson(text) {
let t = String(text).trim().replace(/^```(?:json)?\s*/i, "").replace(/```\s*$/, "");
const a = t.indexOf("{"), b = t.lastIndexOf("}");
if (a === -1 || b === -1) throw new Error("no JSON object in response");
return JSON.parse(t.slice(a, b + 1));
}
/* ---------- pricing / budget ---------- */
export function priceOf(modelId) {
return PRICES[modelId] || null;
}
export function callCost(model, tokensIn, tokensOut) {
const p = priceOf(model);
if (!p) return 0;
return (tokensIn / 1e6) * p.in + (tokensOut / 1e6) * p.out;
}
export function spendFromLedger(ledger, date) {
let usd = 0;
for (const l of ledger) if (l.date === date && typeof l.cost_usd === "number") usd += l.cost_usd;
return usd;
}
/* ---------- event weighting (see docs/design.md) ---------- */
export function eventWeight(sources, facets, threadDays) {
const corrob = sources <= 1 ? 1 : sources === 2 ? 1.6 : 2.0;
const complx = 1 + 0.25 * facets;
const sustention = 1 + 0.1 * Math.min((threadDays || 1) - 1, 5);
return Math.round(corrob * complx * sustention * 100) / 100;
}