Docs/Skills and code/skill script-cohort-analysis

.claude/skills/script-cohort-analysis/scripts/compare.js

Code56 lines3.5 KBSkill: script-cohort-analysis

#!/usr/bin/env node
// Compare two analyzed runs of the SAME cohort (e.g. different rater models).
//   node compare.js --a <runDirA> --b <runDirB> [--out <file.md>]
// Samples are matched by video_id, so the S-id shuffles may differ.
const fs = require('fs');
const path = require('path');
const { readJson, parseArgs, spearman, mean } = require('./lib');

const args = parseArgs(process.argv.slice(2));
if (!args.a || !args.b) { console.error('usage: node compare.js --a <runDirA> --b <runDirB> [--out <file.md>]'); process.exit(1); }
const A = readJson(path.join(path.resolve(args.a), 'analysis.json'));
const B = readJson(path.join(path.resolve(args.b), 'analysis.json'));
const nameA = path.basename(path.resolve(args.a)), nameB = path.basename(path.resolve(args.b));

const vids = Object.values(A.key).map((k) => k.video_id);
const sidOf = (an) => Object.fromEntries(Object.entries(an.key).map(([s, k]) => [k.video_id, s]));
const sa = sidOf(A), sb = sidOf(B);
if (!vids.every((v) => sb[v])) { console.error('Runs do not cover the same videos.'); process.exit(1); }

const f2 = (v) => (v === null || v === undefined ? '—' : v.toFixed(2));
const byId = (an) => Object.fromEntries(an.results.map((r) => [r.id, r]));
const ra = byId(A), rb = byId(B);
const lines = [`# Run comparison: ${nameA} vs ${nameB}`, ''];

lines.push('## Blind prediction accuracy (rho with actual performance)', '', `| Predictor | ${nameA} | ${nameB} |`, '|---|---|---|');
for (const p of A.predictors) {
  const q = B.predictors.find((x) => x.predictor === p.predictor);
  lines.push(`| ${p.predictor} | ${f2(p.rho)} (p ${f2(p.p)}) | ${q ? `${f2(q.rho)} (p ${f2(q.p)})` : '—'} |`);
}

const raterFeats = Object.keys(ra).filter((id) => ra[id].reliability && rb[id] && rb[id].reliability);
const relA = raterFeats.map((id) => ra[id].reliability.rater_rho).filter((v) => v !== null);
const relB = raterFeats.map((id) => rb[id].reliability.rater_rho).filter((v) => v !== null);
lines.push('', '## Inter-rater reliability (within each run)', '',
  `Mean rater-vs-rater rho across ${raterFeats.length} rater-derived features: **${nameA} ${f2(mean(relA))}**, **${nameB} ${f2(mean(relB))}**.`,
  `Unreliable features: ${nameA} ${raterFeats.filter((id) => !ra[id].reliability.ok).length}, ${nameB} ${raterFeats.filter((id) => !rb[id].reliability.ok).length}.`,
  '', `| Feature | ${nameA} rater rho / gap | ${nameB} rater rho / gap | Cross-run rho | ${nameA} verdict | ${nameB} verdict |`, '|---|---|---|---|---|---|');
const cross = [];
for (const id of raterFeats) {
  const xa = vids.map((v) => ra[id].values[sa[v]]), xb = vids.map((v) => rb[id].values[sb[v]]);
  const c = spearman(xa, xb);
  if (c !== null) cross.push(c);
  const x = ra[id].reliability, y = rb[id].reliability;
  const changed = ra[id].verdict !== rb[id].verdict ? ' ⚠' : '';
  lines.push(`| ${id} | ${f2(x.rater_rho)} / ${f2(x.mean_abs_gap)} | ${f2(y.rater_rho)} / ${f2(y.mean_abs_gap)} | ${f2(c)} | ${ra[id].verdict} | ${rb[id].verdict}${changed} |`);
}
lines.push('', `Mean cross-run agreement on sample-level values: **${f2(mean(cross))}** (1 = both runs order the videos identically).`);

const flips = Object.keys(ra).filter((id) => rb[id] && ra[id].verdict !== rb[id].verdict);
lines.push('', `## Verdict changes (${flips.length})`, '');
for (const id of flips) lines.push(`- ${id}: ${ra[id].verdict} → ${rb[id].verdict}`);

const out = lines.join('\n') + '\n';
if (args.out) { fs.writeFileSync(path.resolve(args.out), out, 'utf8'); console.log(`Wrote ${path.resolve(args.out)}`); }
else process.stdout.write(out);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.