Docs/Skills and code/Top-level code
compare-qualification-llm.js
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import Database from "better-sqlite3";
import { config } from "./config.js";
import { callLLM, callOllama, getLLMConfig, getOllamaConfig, parseJsonObject } from "./src/llm.js";
import { buildQualificationInput } from "./src/analysis/channel-evidence.js";
import { normalizeLlmResult, qualificationPrompt } from "./src/analysis/channel-qualification-store.js";
import { GeminiError, callGeminiWithRetry, explainGeminiError } from "./src/gemini.js";
import { listFreeModels, postChat } from "./test-openrouter.js";
import { postChat as postGroqChat } from "./test-groq.js";
// ============================================================
// QUALIFICATION MODEL COMPARISON (read-only; writes only a report file)
//
// Sends the same batch that `npm run qualification:llm` would send, built from
// the same prompt and input packets, to:
// 1. Ollama (the model the stage used), 2. Gemma 4 on the Google Gemini API,
// 3. optionally some OpenRouter free models (--models=).
// It compares speed, whether the reply survives the stage's own parser, and how
// the labels line up with the stored Gemma results.
//
// node compare-qualification-llm.js
// node compare-qualification-llm.js --gemini=gemma-4-31b-it,gemma-4-31b-it@minimal,gemma-4-26b-a4b-it
// node compare-qualification-llm.js --models=nvidia/nemotron-3-ultra-550b-a55b:free
// node compare-qualification-llm.js --channels=32 --skip-ollama
// node compare-qualification-llm.js --skip-ollama --skip-gemini --groq=openai/gpt-oss-120b,qwen/qwen3.8-27b
//
// A Gemini entry is <model>[@option[+option]]. Options: minimal | high (thinking level) and
// system (send the system prompt as systemInstruction instead of one combined prompt string),
// json (Gemini JSON response mode).
// ============================================================
const args = process.argv.slice(2);
const getOption = (name) => args.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3);
const hasFlag = (name) => args.includes(`--${name}`);
const SAMPLE_SIZE = Math.max(1, Math.floor(Number(getOption("channels") ?? config.qualification.llm.batchSize)));
const MAX_TOKENS = Number(getOption("max-tokens") ?? 16000);
const TEMPERATURE = Number(getOption("temperature") ?? 0.1);
const TIMEOUT_MS = Number(getOption("timeout-s") ?? 300) * 1000;
const RATE_LIMIT_ATTEMPTS = 3;
const DEFAULT_GEMINI = ["gemma-4-31b-it", "gemma-4-31b-it@minimal", "gemma-4-26b-a4b-it", "gemma-4-31b-it@system"];
const splitList = (value) => value.split(",").map((id) => id.trim()).filter(Boolean);
const modelIds = splitList(getOption("models") ?? "");
const groqIds = splitList(getOption("groq") ?? "");
const geminiSpecs = hasFlag("skip-gemini") ? [] : splitList(getOption("gemini") ?? DEFAULT_GEMINI.join(","));
// Cycled so the sample covers every stored label, not just the most common one.
const CLASS_PATTERN = [
"core", "adjacent", "irrelevant", "core", "adjacent", "irrelevant", "core", "unknown",
"core", "adjacent", "irrelevant", "core", "adjacent", "irrelevant", "core", "unknown"
];
const mean = (values) => {
const valid = values.filter(Number.isFinite);
return valid.length ? valid.reduce((sum, value) => sum + value, 0) / valid.length : null;
};
const round = (value, digits = 2) => (Number.isFinite(value) ? Number(value.toFixed(digits)) : null);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
// ------------------------------------------------------------
// Load channels exactly the way 25-qualify-channels-with-llm.js does.
// ------------------------------------------------------------
const db = new Database(config.dbPath, { readonly: true, fileMustExist: true });
const promptVersion = config.qualification.promptVersion;
const channelRows = db.prepare(`
SELECT c.*,
COALESCE(m.name,c.name) AS resolved_name,
COALESCE(m.description,c.description) AS resolved_description,
COALESCE(m.keywords_json,c.keywords_json) AS resolved_keywords_json,
COALESCE(m.subscriber_count,c.subscriber_count) AS resolved_subscriber_count,
COALESCE(m.subscriber_count_text,c.subscriber_count_text) AS resolved_subscriber_count_text,
COALESCE(m.video_count,c.video_count) AS resolved_video_count,
COALESCE(m.joined_date,c.joined_date) AS resolved_joined_date
FROM channels c LEFT JOIN channel_metadata m ON m.channel_url=c.channel_url
`).all();
const videosByChannel = new Map();
const videoRows = db.prepare(`
SELECT cv.channel_url, v.video_id, v.title, v.view_count, v.published_at, v.published_at_normalized, v.published_at_precision
FROM channel_videos cv JOIN videos v ON v.video_id=cv.video_id
UNION
SELECT v.channel_url, v.video_id, v.title, v.view_count, v.published_at, v.published_at_normalized, v.published_at_precision
FROM videos v WHERE v.channel_url IS NOT NULL
`).all();
for (const video of videoRows) {
if (!video.channel_url) continue;
if (!videosByChannel.has(video.channel_url)) videosByChannel.set(video.channel_url, new Map());
const map = videosByChannel.get(video.channel_url);
const prior = map.get(video.video_id);
if (!prior || (Number(video.view_count) || 0) > (Number(prior.view_count) || 0)) map.set(video.video_id, video);
}
const sourceChannels = new Map(channelRows.map((row) => {
let keywords = [];
try {
const parsed = JSON.parse(row.resolved_keywords_json ?? "[]");
keywords = Array.isArray(parsed) ? parsed : [];
} catch {
// Unparseable keywords are treated as none, as in the stage.
}
return [row.channel_url, {
channel_url: row.channel_url, name: row.resolved_name, description: row.resolved_description,
keywords, keywords_json: row.resolved_keywords_json,
subscriber_count: row.resolved_subscriber_count, subscriber_count_text: row.resolved_subscriber_count_text,
video_count: row.resolved_video_count, joined_date: row.resolved_joined_date,
is_verified: row.is_verified, is_artist: row.is_artist,
videos: [...(videosByChannel.get(row.channel_url)?.values() ?? [])]
}];
}));
// Stored Gemma results are the baseline. Only channels whose current input hash
// equals the stored one are used, so every model sees exactly what Gemma saw.
const storedRows = db.prepare(`
SELECT channel_url, input_hash, classification, confidence, relevance_score, focus_specificity,
channel_type, focus_summary, rationale, topic_tags_json, model, created_at
FROM channel_qualification_llm_results
WHERE processing_status='succeeded' AND prompt_version=?
ORDER BY created_at DESC
`).all(promptVersion);
const storedByUrl = new Map();
for (const row of storedRows) if (!storedByUrl.has(row.channel_url)) storedByUrl.set(row.channel_url, row);
const pools = new Map();
let hashMismatches = 0;
for (const [url, stored] of storedByUrl) {
const channel = sourceChannels.get(url);
if (!channel || !stored.classification) continue;
const input = buildQualificationInput(channel);
if (input.inputHash !== stored.input_hash) {
hashMismatches += 1;
continue;
}
if (!pools.has(stored.classification)) pools.set(stored.classification, []);
pools.get(stored.classification).push({ channel, ...input, stored });
}
// Deterministic pseudo-random order (not alphabetical) so the sample is not cherry-picked.
const shuffleKey = (item) => crypto.createHash("sha256").update(item.channel.channel_url).digest("hex");
for (const pool of pools.values()) pool.sort((a, b) => shuffleKey(a).localeCompare(shuffleKey(b)));
const sample = [];
for (let i = 0; sample.length < SAMPLE_SIZE && i < SAMPLE_SIZE * 4; i++) {
const wanted = CLASS_PATTERN[i % CLASS_PATTERN.length];
const item = pools.get(wanted)?.shift();
if (item) sample.push(item);
}
if (!sample.length) {
throw new Error("No channels with a stored Gemma result and an unchanged input hash were found.");
}
// ------------------------------------------------------------
// Build the identical request the stage sends.
// ------------------------------------------------------------
const system = qualificationPrompt();
const user = `Classify these channels as one batch. Return one result for every exact channel_url.\n\n${JSON.stringify({ channels: sample.map((item) => item.packet) })}`;
// src/llm.js callLLM() sends system + user as a single prompt string; do the same here.
const prompt = [system.trim(), user.trim()].join("\n\n");
// ------------------------------------------------------------
// Callers
// ------------------------------------------------------------
async function runOllama() {
const { model } = getOllamaConfig();
const started = Date.now();
const content = await callOllama(prompt, { stream: false });
return { model, content, seconds: (Date.now() - started) / 1000, attempts: 1 };
}
async function runOpenRouter(model) {
const messages = [{ role: "user", content: prompt }];
let lastError;
for (let attempt = 1; attempt <= RATE_LIMIT_ATTEMPTS; attempt++) {
const started = Date.now();
try {
const res = await postChat({ model, messages, stream: false, maxTokens: MAX_TOKENS, temperature: TEMPERATURE, timeoutMs: TIMEOUT_MS });
const data = await res.json();
if (data.error) {
throw Object.assign(new Error(JSON.stringify(data.error).slice(0, 500)), { status: data.error.code });
}
const choice = data.choices?.[0];
return {
model,
content: choice?.message?.content ?? "",
seconds: (Date.now() - started) / 1000,
attempts: attempt,
finishReason: choice?.finish_reason ?? null,
usage: data.usage ?? null,
reasoningChars: String(choice?.message?.reasoning ?? "").length
};
} catch (error) {
lastError = error;
if (error.status === 429 && attempt < RATE_LIMIT_ATTEMPTS) {
const waitSeconds = 20 * attempt;
console.warn(` 429 from ${model}; retrying in ${waitSeconds}s (attempt ${attempt}/${RATE_LIMIT_ATTEMPTS})`);
await sleep(waitSeconds * 1000);
continue;
}
break;
}
}
throw lastError;
}
// Groq's free tier allows roughly 7-8k tokens per minute per model, far below one 16-channel request
// (~15k tokens), so the sample is sent as several smaller requests with the same system prompt and
// user template, then merged into one reply. seconds is API time only; the waits are not counted.
const GROQ_BATCH = Math.max(1, Math.floor(Number(getOption("groq-batch") ?? 4)));
const GROQ_WINDOW_S = 65;
const GROQ_MAX_TOKENS = Number(getOption("groq-max-tokens") ?? 4000); // qwen3.8-27b allows only 1000 output tokens per minute
async function groqRequest(model, content) {
let lastError;
for (let attempt = 1; attempt <= RATE_LIMIT_ATTEMPTS; attempt++) {
const started = Date.now();
try {
// Reasoning models (gpt-oss) spend output tokens on thinking; keep it low so the JSON fits.
const extra = model.startsWith("openai/gpt-oss") ? { reasoning_effort: "low" } : {};
const data = await postGroqChat({
model, messages: [{ role: "user", content }], maxTokens: Math.min(MAX_TOKENS, GROQ_MAX_TOKENS),
temperature: TEMPERATURE, timeoutMs: TIMEOUT_MS, extra
});
return { data, seconds: (Date.now() - started) / 1000, attempts: attempt };
} catch (error) {
lastError = error;
if ([413, 429].includes(error.status) && attempt < RATE_LIMIT_ATTEMPTS) {
const waitSeconds = Math.min(Math.max(error.retryAfterSeconds ?? 0, GROQ_WINDOW_S), 90);
console.warn(` ${error.status} from ${model}; retrying in ${waitSeconds}s (attempt ${attempt}/${RATE_LIMIT_ATTEMPTS})`);
await sleep(waitSeconds * 1000);
continue;
}
break;
}
}
throw lastError;
}
async function runGroq(model) {
const rows = [];
const total = { seconds: 0, attempts: 0, prompt_tokens: 0, completion_tokens: 0, reasoningChars: 0 };
const finishReasons = [];
for (let i = 0; i < sample.length; i += GROQ_BATCH) {
if (i > 0) await sleep(GROQ_WINDOW_S * 1000);
const chunk = sample.slice(i, i + GROQ_BATCH);
const chunkUser = `Classify these channels as one batch. Return one result for every exact channel_url.\n\n${JSON.stringify({ channels: chunk.map((item) => item.packet) })}`;
const { data, seconds, attempts } = await groqRequest(model, [system.trim(), chunkUser.trim()].join("\n\n"));
const choice = data.choices?.[0];
total.seconds += seconds;
total.attempts += attempts;
total.prompt_tokens += data.usage?.prompt_tokens ?? 0;
total.completion_tokens += data.usage?.completion_tokens ?? 0;
total.reasoningChars += String(choice?.message?.reasoning ?? "").length;
finishReasons.push(choice?.finish_reason ?? "?");
console.log(` chunk ${i / GROQ_BATCH + 1}/${Math.ceil(sample.length / GROQ_BATCH)}: ${round(seconds, 1)}s, finish ${choice?.finish_reason}`);
try {
const parsed = parseJsonObject(choice?.message?.content ?? "");
rows.push(...(Array.isArray(parsed) ? parsed : parsed?.results ?? []));
} catch (error) {
// An unparseable chunk leaves its channels missing; evaluate() reports them as such.
console.warn(` chunk unparseable: ${String(error.message).split("\n")[0].slice(0, 120)}`);
}
}
return {
model, content: JSON.stringify({ results: rows }), seconds: total.seconds, attempts: total.attempts,
finishReason: [...new Set(finishReasons)].join("+"), usage: total, reasoningChars: total.reasoningChars
};
}
// The exact call stage 25 makes: whatever backend src/llm.js picks, with its own defaults and fallback.
async function runCallLLM() {
const started = Date.now();
const response = await callLLM({ system, user, stream: false });
return {
model: response.model, provider: response.provider, fellBack: response.fellBack, content: response.content,
seconds: (Date.now() - started) / 1000, attempts: 1, usage: response.usage ?? null
};
}
async function runGemini(spec) {
const [model, optionText = ""] = spec.split("@");
const options = optionText.split("+").filter(Boolean);
const thinkingLevel = options.find((option) => option === "minimal" || option === "high") ?? "";
const useSystemInstruction = options.includes("system");
const jsonMode = options.includes("json");
const result = await callGeminiWithRetry({
// Default mirrors src/llm.js callLLM(): system and user joined into ONE prompt string.
system: useSystemInstruction ? system : "",
user: useSystemInstruction ? user : prompt,
model, stream: false, maxOutputTokens: MAX_TOKENS, temperature: TEMPERATURE, thinkingLevel, jsonMode, timeoutMs: TIMEOUT_MS
}, {
maxRetries: 2,
maxWaitSeconds: 90,
onWait: ({ error, waitSeconds }) => console.warn(` ${explainGeminiError(error)}
Waiting ${waitSeconds}s, then retrying`)
});
return {
model, content: result.content, seconds: result.seconds, attempts: result.attempts,
finishReason: result.finishReason, usage: result.usage
};
}
// ------------------------------------------------------------
// Evaluate a reply with the stage's own parser and normalizer.
// ------------------------------------------------------------
function evaluate(run) {
const outcome = { parseError: null, returned: 0, valid: 0, invalid: [], byUrl: new Map() };
let parsed;
try {
parsed = parseJsonObject(run.content);
} catch (error) {
outcome.parseError = String(error.message).split("\n")[0].slice(0, 200);
return outcome;
}
const rows = Array.isArray(parsed) ? parsed : parsed?.results;
const rawByUrl = new Map();
for (const row of Array.isArray(rows) ? rows : []) {
const url = String(row?.channel_url ?? row?.channelUrl ?? "").trim();
if (url && !rawByUrl.has(url)) rawByUrl.set(url, row);
}
for (const item of sample) {
const raw = rawByUrl.get(item.channel.channel_url);
if (!raw) {
outcome.invalid.push({ channel: item.channel.name, error: "missing from reply" });
continue;
}
outcome.returned += 1;
try {
outcome.byUrl.set(item.channel.channel_url, normalizeLlmResult(raw, item.channel.channel_url));
outcome.valid += 1;
} catch (error) {
outcome.invalid.push({ channel: item.channel.name, error: error.message.slice(0, 160) });
}
}
return outcome;
}
const tagSet = (tags) => new Set((tags ?? []).map((tag) => String(tag).trim().toLowerCase()).filter(Boolean));
function jaccard(a, b) {
if (!a.size && !b.size) return null;
const intersection = [...a].filter((tag) => b.has(tag)).length;
return intersection / new Set([...a, ...b]).size;
}
function storedAsResult(item) {
const s = item.stored;
let tags = [];
try { tags = JSON.parse(s.topic_tags_json ?? "[]"); } catch { tags = []; }
return { classification: s.classification, confidence: s.confidence, topic_tags: tags, rationale: s.rationale, focus_summary: s.focus_summary };
}
function summarize(label, run, outcome, liveGemmaOutcome) {
const valid = sample.filter((item) => outcome.byUrl.has(item.channel.channel_url));
const results = valid.map((item) => outcome.byUrl.get(item.channel.channel_url));
const agreeWith = (reference) => {
const comparable = valid.filter((item) => reference(item));
if (!comparable.length) return null;
const same = comparable.filter((item) => reference(item).classification === outcome.byUrl.get(item.channel.channel_url).classification).length;
return `${same}/${comparable.length}`;
};
return {
run: label,
secs: round(run?.seconds, 1),
tries: run?.attempts ?? null,
finish: run?.finishReason ?? "",
in_tokens: run?.usage?.promptTokenCount ?? run?.usage?.prompt_tokens ?? "",
out_tokens: run?.usage?.candidatesTokenCount ?? run?.usage?.completion_tokens ?? "",
think_tokens: run?.usage?.thoughtsTokenCount ?? "",
json_ok: outcome.parseError ? "NO" : "yes",
rows: `${outcome.returned}/${sample.length}`,
valid: `${outcome.valid}/${sample.length}`,
vs_stored_gemma: agreeWith((item) => storedAsResult(item)),
vs_live_gemma: liveGemmaOutcome && outcome !== liveGemmaOutcome
? agreeWith((item) => liveGemmaOutcome.byUrl.get(item.channel.channel_url))
: "",
mean_conf: round(mean(results.map((r) => r.confidence))),
conf_ge_75: results.length ? `${results.filter((r) => r.confidence >= 0.75).length}/${results.length}` : "",
tags_per_ch: round(mean(results.map((r) => r.topic_tags.length)), 1),
tag_jaccard_vs_stored: round(mean(valid.map((item) => jaccard(
tagSet(outcome.byUrl.get(item.channel.channel_url).topic_tags),
tagSet(storedAsResult(item).topic_tags)
))))
};
}
// ------------------------------------------------------------
// Run
// ------------------------------------------------------------
console.log(`Database: ${config.dbPath}`);
console.log(`Ecosystem: ${config.ecosystem.name} | prompt ${promptVersion}`);
console.log(`Sample: ${sample.length} channels (${hashMismatches} stored results skipped because their input changed since)`);
console.log(`Request: ${prompt.length.toLocaleString()} characters, one prompt string, temperature ${TEMPERATURE} on Gemini/OpenRouter (Ollama: its default), max output tokens ${MAX_TOKENS}`);
console.log("Sample labels per stored Gemma result:", Object.fromEntries(
[...new Set(sample.map((s) => s.stored.classification))].map((label) => [label, sample.filter((s) => s.stored.classification === label).length])
));
const runs = [];
if (!hasFlag("skip-ollama")) {
const { model } = getOllamaConfig();
console.log(`\n[1] Ollama ${model} (live, same path as the stage) ...`);
try {
const run = await runOllama();
runs.push({ label: `ollama ${model} (live)`, run, outcome: evaluate(run), isGemmaLive: true });
console.log(` done in ${round(run.seconds, 1)}s`);
} catch (error) {
console.error(` FAILED: ${error.message}`);
}
}
if (hasFlag("via-llm")) {
const config = getLLMConfig();
console.log(`
[${runs.length + 1}] src/llm.js callLLM() as stage 25 calls it (primary ${config.provider} ${config.model}, fallback ${config.fallback?.model ?? "none"}) ...`);
try {
const run = await runCallLLM();
runs.push({ label: `callLLM ${run.provider} ${run.model}${run.fellBack ? " (fell back)" : ""}`, run, outcome: evaluate(run) });
console.log(` answered by ${run.provider} ${run.model} in ${round(run.seconds, 1)}s`);
} catch (error) {
console.error(` FAILED: ${error.message}`);
runs.push({ label: "callLLM", run: null, outcome: { parseError: error.message.slice(0, 200), returned: 0, valid: 0, invalid: [], byUrl: new Map() }, failed: error.message });
}
}
for (const spec of geminiSpecs) {
console.log(`
[${runs.length + 1}] Gemini API ${spec} ...`);
try {
const run = await runGemini(spec);
runs.push({ label: `gemini ${spec}`, run, outcome: evaluate(run) });
console.log(` done in ${round(run.seconds, 1)}s (tries ${run.attempts})`);
} catch (error) {
const note = explainGeminiError(error);
console.error(` FAILED: ${note}`);
runs.push({ label: `gemini ${spec}`, run: null, outcome: { parseError: note.slice(0, 200), returned: 0, valid: 0, invalid: [], byUrl: new Map() }, failed: note });
// A daily limit will fail every remaining Gemini run; stop spending time on them.
if (error instanceof GeminiError && error.kind === "daily-limit") break;
}
}
for (const model of groqIds) {
console.log(`
[${runs.length + 1}] Groq ${model} ...`);
try {
const run = await runGroq(model);
runs.push({ label: `groq ${model}`, run, outcome: evaluate(run) });
console.log(` done in ${round(run.seconds, 1)}s (tries ${run.attempts})`);
} catch (error) {
console.error(` FAILED: ${error.message}`);
runs.push({ label: `groq ${model}`, run: null, outcome: { parseError: String(error.message).slice(0, 200), returned: 0, valid: 0, invalid: [], byUrl: new Map() }, failed: error.message });
}
}
const freeIds = modelIds.length ? new Set((await listFreeModels()).map((m) => m.id)) : new Set();
for (const model of modelIds) {
console.log(`\n[${runs.length + 1}] OpenRouter ${model} ...`);
if (!freeIds.has(model)) {
console.warn(" not in the current free-model list; skipping");
continue;
}
try {
const run = await runOpenRouter(model);
runs.push({ label: `openrouter ${model}`, run, outcome: evaluate(run) });
console.log(` done in ${round(run.seconds, 1)}s`);
} catch (error) {
console.error(` FAILED: ${error.message}`);
runs.push({ label: `openrouter ${model}`, run: null, outcome: { parseError: String(error.message).slice(0, 200), returned: 0, valid: 0, invalid: [], byUrl: new Map() }, failed: error.message });
}
}
const liveGemma = runs.find((r) => r.isGemmaLive)?.outcome ?? null;
// Reference row: how the stored run itself looked, timed from the full historical run.
const storedRun = db.prepare(`
SELECT started_at, finished_at, requested_channel_count, batch_size FROM channel_qualification_llm_runs
WHERE model=? AND prompt_version=? AND status='completed' ORDER BY started_at DESC LIMIT 1
`).get(getOllamaConfig().model, promptVersion);
const storedSecondsPerBatch = storedRun
? ((Date.parse(storedRun.finished_at) - Date.parse(storedRun.started_at)) / 1000) / Math.ceil(storedRun.requested_channel_count / storedRun.batch_size)
: null;
console.log("\n==================== SUMMARY ====================");
if (storedSecondsPerBatch != null) {
console.log(`Stored production run (${getOllamaConfig().model}): ${round(storedSecondsPerBatch, 1)} s per batch of ${storedRun.batch_size}, averaged over the whole run.`);
}
console.table(runs.map(({ label, run, outcome }) => summarize(label, run, outcome, liveGemma)));
for (const { label, outcome, failed } of runs) {
if (failed) console.log(`${label}: request failed -> ${failed.slice(0, 300)}`);
else if (outcome.parseError) console.log(`${label}: unparseable reply -> ${outcome.parseError}`);
else if (outcome.invalid.length) console.log(`${label}: ${outcome.invalid.length} invalid rows ->`, outcome.invalid.slice(0, 4));
}
// Channels where any run disagrees with the stored Gemma label.
const short = { core: "core", adjacent: "adj", irrelevant: "irr", unknown: "unk" };
const disagreements = sample.map((item) => {
const row = { channel: item.channel.name.slice(0, 34), stored: short[item.stored.classification] ?? item.stored.classification };
let differs = false;
for (const { label, outcome } of runs) {
const result = outcome.byUrl.get(item.channel.channel_url);
const key = label.replace(/^(ollama|openrouter|gemini|groq|callLLM) /, "").replace(/ \(live\)$/, " live").slice(0, 26);
row[key] = result ? (short[result.classification] ?? result.classification) : "-";
if (result && result.classification !== item.stored.classification) differs = true;
}
return differs ? row : null;
}).filter(Boolean);
console.log(`\nChannels where any run differs from the stored Gemma label: ${disagreements.length}/${sample.length}`);
if (disagreements.length) console.table(disagreements);
const reportPath = path.join(process.cwd(), "reports", `qualification-model-comparison-${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`);
fs.mkdirSync(path.dirname(reportPath), { recursive: true });
fs.writeFileSync(reportPath, JSON.stringify({
generated_at: new Date().toISOString(),
ecosystem: config.ecosystem.name,
prompt_version: promptVersion,
prompt_chars: prompt.length,
settings: { temperature: TEMPERATURE, max_tokens: MAX_TOKENS, sample_size: sample.length, hash_mismatches_skipped: hashMismatches },
stored_seconds_per_batch: storedSecondsPerBatch,
sample: sample.map((item) => ({ channel_url: item.channel.channel_url, name: item.channel.name, stored: storedAsResult(item) })),
runs: runs.map(({ label, run, outcome, failed }) => ({
label, failed: failed ?? null, seconds: run?.seconds ?? null, attempts: run?.attempts ?? null,
finish_reason: run?.finishReason ?? null, usage: run?.usage ?? null, reasoning_chars: run?.reasoningChars ?? null,
parse_error: outcome.parseError, invalid: outcome.invalid,
results: Object.fromEntries(outcome.byUrl),
raw_reply: run?.content ?? null
}))
}, null, 2));
console.log(`\nFull detail (labels, rationales, tags, raw replies): ${reportPath}`);
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.