scripts/lib.js
// Shared utilities for emergence scripts. Zero dependencies by design, mirroring
// the xray engine conventions (same auth resolution, same arg parser, same tone).
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import crypto from "node:crypto";
export const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..");
export const MONO_ROOT = path.resolve(REPO_ROOT, "..");
export const DATA_ROOT = process.env.EMERGENCE_DATA || path.join(MONO_ROOT, "emergence-data");
export { PROMPT_VERSION, prevDate } from "../pipeline/util.js";
export const AUTH_FILE = path.join(os.homedir(), ".local/share/opencode/auth.json");
export function getCloudflareAuth() {
const token = process.env.CLOUDFLARE_API_TOKEN || null;
const account = process.env.CLOUDFLARE_ACCOUNT_ID || null;
if (token) return { token, account, source: "env" };
try {
const raw = JSON.parse(fs.readFileSync(AUTH_FILE, "utf8"));
const key = raw?.["cloudflare-workers-ai"]?.key || null;
if (key) return { token: key, account, source: "auth.json" };
} catch {}
return { token: null, account, source: "none" };
}
// The controlled YAML subset used by sources/sources.yaml and ontology/topics.yaml:
// list of maps at indent 0 ("- id: x"), string fields at indent 2, and a
// comma-separated string-list field ("subtopics: a, b, c").
export function parseRegistryYaml(text) {
const items = [];
let cur = null;
for (const raw of String(text).split("\n")) {
const line = raw.replace(/\r$/, "");
if (!line.trim() || /^\s*#/.test(line)) continue;
const indent = line.length - line.trimStart().length;
const t = line.trim();
if (indent === 0 && t.startsWith("- ")) {
cur = {};
items.push(cur);
const rest = t.slice(2).trim();
const ci = rest.indexOf(":");
if (ci > -1) cur[rest.slice(0, ci).trim()] = rest.slice(ci + 1).trim();
continue;
}
const ci = t.indexOf(":");
if (ci === -1 || indent !== 2) continue;
const key = t.slice(0, ci).trim();
const val = t.slice(ci + 1).trim();
if (val === "") continue;
cur[key] = key === "subtopics" ? val.split(",").map((s) => s.trim()).filter(Boolean) : val;
}
return items;
}
export function loadConfig() {
return JSON.parse(fs.readFileSync(path.join(REPO_ROOT, "scripts/config.json"), "utf8"));
}
export function loadSources() {
return parseRegistryYaml(fs.readFileSync(path.join(REPO_ROOT, "sources/sources.yaml"), "utf8"));
}
export function loadTopics() {
return parseRegistryYaml(fs.readFileSync(path.join(REPO_ROOT, "ontology/topics.yaml"), "utf8"));
}
export function topicListForPrompt() {
const t = loadTopics().filter((x) => x.id !== "other");
return t.map((x) => `${x.id}: ${x.subtopics.join(", ")}`).join("\n");
}
export function utcDate(d = new Date()) {
return d.toISOString().slice(0, 10);
}
export function nowIso() {
return new Date().toISOString();
}
export function hashId(prefix, content) {
const h = crypto.createHash("sha256").update(content).digest("hex").slice(0, 12);
return `${prefix}_${h}`;
}
export function appendJsonl(filePath, obj) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.appendFileSync(filePath, JSON.stringify(obj) + "\n", "utf8");
}
export function readJsonl(filePath) {
if (!fs.existsSync(filePath)) return [];
return fs.readFileSync(filePath, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
}
export function readAllJsonl(dir, subdirPattern) {
const out = [];
if (!fs.existsSync(dir)) return out;
for (const day of fs.readdirSync(dir).sort()) {
const p = path.join(dir, day, subdirPattern);
if (fs.existsSync(p)) out.push(...readJsonl(p));
}
return out;
}
export function fmtUsd(x) {
return "$" + x.toFixed(x < 1 ? 3 : 2);
}
export function parseArgs(argv, spec) {
const out = {};
const positional = [];
for (const k of Object.keys(spec)) out[k] = spec[k].default;
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a.startsWith("--")) {
const key = a.slice(2).replace(/-/g, "_");
const s = spec[key];
if (!s) { console.error(`Unknown flag --${key}`); process.exit(2); }
if (s.takes === "none") out[key] = true;
else {
let val = argv[i + 1];
if (val === undefined || val.startsWith("--")) {
if (s.takes === "optional") val = true;
else { console.error(`--${key} requires a value`); process.exit(2); }
out[key] = val;
} else {
out[key] = val;
i++;
}
}
} else positional.push(a);
}
return { args: out, positional };
}
/* ---------- Workers AI client (distill + council share it) ---------- */
export const PRICES = {
"@cf/zai-org/glm-5.3-flash": { in: 0.15, out: 0.5 },
"@cf/zai-org/glm-5.3": { in: 1.4, out: 4.4 },
};
export function priceOf(modelId) {
return PRICES[modelId] || null;
}
export async function runModel(modelId, messages, { temperature = 0.4, max_tokens = 1024 } = {}) {
const auth = getCloudflareAuth();
if (!auth.token) {
throw new Error("No Cloudflare API token found. Set CLOUDFLARE_API_TOKEN or populate " + AUTH_FILE);
}
if (!auth.account) throw new Error("CLOUDFLARE_ACCOUNT_ID is not set.");
const url = `https://api.cloudflare.com/client/v4/accounts/${auth.account}/ai/run/${modelId}`;
const res = await fetch(url, {
method: "POST",
headers: { Authorization: `Bearer ${auth.token}`, "Content-Type": "application/json" },
body: JSON.stringify({ messages, temperature, max_tokens }),
});
if (!res.ok) throw new Error(`API error ${res.status}: ${(await res.text()).slice(0, 300)}`);
const data = await res.json();
if (!data.success) throw new Error(`API failure: ${JSON.stringify(data.errors || data).slice(0, 300)}`);
const text = data.result?.choices?.[0]?.message?.content ?? data.result?.response ?? "";
const usage = data.result?.usage ?? {};
const tokensIn = usage.prompt_tokens ?? null;
const tokensOut = usage.completion_tokens ?? null;
let cost = null;
const p = priceOf(modelId);
if (p && tokensIn != null && tokensOut != null) {
cost = (tokensIn / 1e6) * p.in + (tokensOut / 1e6) * p.out;
}
return { text, tokensIn, tokensOut, cost };
}
/* ---------- ledger + budget guard ---------- */
export function ledgerPath() {
return path.join(DATA_ROOT, "ledger.jsonl");
}
export function logSpend({ script, model, tokensIn, tokensOut, cost }) {
appendJsonl(ledgerPath(), {
date: utcDate(), at: nowIso(), script, model,
tokens_in: tokensIn, tokens_out: tokensOut, cost_usd: cost,
});
}
export function spendToday() {
const today = utcDate();
let usd = 0;
for (const l of readJsonl(ledgerPath())) {
if (l.date === today && typeof l.cost_usd === "number") usd += l.cost_usd;
}
return usd;
}
export function budgetCheck(config, purpose) {
const spent = spendToday();
const remaining = config.daily_budget_usd - spent;
if (remaining <= 0) {
console.error(`[budget] ${purpose} skipped: ${fmtUsd(spent)} spent today, cap ${fmtUsd(config.daily_budget_usd)}.`);
return { ok: false, spent, remaining: 0 };
}
return { ok: true, spent, remaining };
}