scripts/converge.js
#!/usr/bin/env node
// converge.js — phase 2, step 1: deterministic clustering of the active-bench records
// into cross-model themes within each domain, for human/agent adjudication.
// Output: docs/corpus/clusters.json + a printed summary of top clusters with claims.
import fs from "node:fs";
import path from "node:path";
import { REPO_ROOT } from "./lib.js";
const INDEX = path.join(REPO_ROOT, "docs/corpus/all-records.jsonl");
const OUT = path.join(REPO_ROOT, "docs/corpus/clusters.json");
const DEPRECATED = "meta-llama-3-1-8b-instruct-fp8";
const STOP = new Set("a an the of in on at to for and or is are was were be been it its this that with as by from not no but if then than so such their there here what which who whom how why more most other some any can could would should will may might do does did have has had about into over under also very".split(" "));
const recs = fs.readFileSync(INDEX, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
const active = recs.filter((r) => r._model !== DEPRECATED);
const toks = (r) => {
const t = new Set(
(r.claim + " " + (r.tags || []).join(" ") + " " + (r.conditions || ""))
.toLowerCase()
.replace(/[^a-z0-9\s-]/g, " ")
.split(/\s+/)
.filter((w) => w.length > 2 && !STOP.has(w))
);
return t;
};
const jac = (a, b) => {
let inter = 0;
for (const x of a) if (b.has(x)) inter++;
return inter / (a.size + b.size - inter);
};
// Cluster within each domain (across lenses), greedy single-pass
const byDomain = new Map();
for (const r of active) {
if (!byDomain.has(r.elicitation.domain)) byDomain.set(r.elicitation.domain, []);
byDomain.get(r.elicitation.domain).push({ r, t: toks(r) });
}
const clusters = [];
for (const [domain, list] of byDomain) {
const local = [];
for (const item of list) {
let best = null, bestScore = 0;
for (const c of local) {
const s = jac(item.t, c.t);
if (s > bestScore) { bestScore = s; best = c; }
}
if (best && bestScore >= 0.16) {
best.items.push(item);
for (const x of item.t) best.t.add(x);
} else local.push({ items: [item], t: new Set(item.t) });
}
for (const c of local) {
const models = [...new Set(c.items.map((i) => i.r._model))];
if (c.items.length >= 2) {
clusters.push({
cluster_id: `${domain}-${clusters.filter((x) => x.cluster_id.startsWith(domain)).length + 1}`,
domain,
size: c.items.length,
n_models: models.length,
record_ids: c.items.map((i) => i.r.id),
items: c.items.map((i) => ({
model: i.r._model, id: i.r.id, lens: i.r.elicitation.lens,
stance: i.r.stance_type, claim: i.r.claim,
})),
});
}
}
}
clusters.sort((a, b) => b.size - a.size || b.n_models - a.n_models);
fs.writeFileSync(OUT, JSON.stringify({ generated: new Date().toISOString(), clusters }, null, 2));
// Print top clusters for adjudication
const top = clusters.filter((c) => c.n_models >= 2).slice(0, 18);
console.log(`${clusters.length} clusters (>=2 records, single-model clusters excluded); ${clusters.filter((c) => c.n_models >= 2).length} are cross-model`);
console.log(`coverage of adjudication-ready clusters: ${clusters.filter((c) => c.n_models >= 2).reduce((s, c) => s + c.size, 0)} of ${active.length} active records\n`);
for (const c of top) {
console.log(`### ${c.cluster_id} (${c.size} recs, ${c.n_models} models)`);
for (const i of c.items.slice(0, 8)) {
console.log(` [${i.model.replace(/^(google|openai|zai-org|qwen)-/, "")} ${i.lens}] ${i.claim.slice(0, 110)}`);
}
if (c.items.length > 8) console.log(` ... +${c.items.length - 8} more`);
console.log("");
}