Docs/Skills and code/skill script-cohort-analysis
.claude/skills/script-cohort-analysis/scripts/analyze.js
What the file says about itself
Phase 3: unseal the key, test every pre-registered feature against performance, measure reliability, find stumped samples, build the rule scorecard, update ledger + rule history.
#!/usr/bin/env node
// Phase 3: unseal the key, test every pre-registered feature against performance, measure
// reliability, find stumped samples, build the rule scorecard, update ledger + rule history.
//
// node analyze.js --run <runDir> [--no-ledger] [--calibration <dir>]
//
// Writes <runDir>/analysis.json, report-data.md, rule-scorecard.md, rule-scorecard.json.
const fs = require('fs');
const path = require('path');
const { readJson, writeJson, sha256, parseArgs, mean, ranks, spearman, permTest } = require('./lib');
const { raterFeatures } = require('./raterFeatures');
const { DIMS } = require('./checklist');
const { buildScorecard } = require('./scorecard');
const args = parseArgs(process.argv.slice(2));
if (!args.run) { console.error('usage: node analyze.js --run <runDir> [--no-ledger] [--calibration <dir>]'); process.exit(1); }
const runDir = path.resolve(args.run);
const manifest = readJson(path.join(runDir, 'manifest.json'));
const preregText = fs.readFileSync(path.join(runDir, 'prereg.json'), 'utf8').trimEnd();
const prereg = JSON.parse(preregText);
const hashOk = fs.readFileSync(path.join(runDir, 'prereg.sha256'), 'utf8').trim() === sha256(preregText);
const key = readJson(path.join(runDir, 'sealed', 'key.json'));
const det = readJson(path.join(runDir, 'features.json'));
const sids = manifest.samples;
const n = sids.length;
const winShare = (w, g) => (g ? w / g : null);
// ---------- per-rater features ----------
const raterFiles = {};
for (const sid of sids) {
raterFiles[sid] = fs.readdirSync(path.join(runDir, 'scores'))
.filter((f) => new RegExp(`^${sid}\\.r\\d+\\.json$`).test(f)).sort()
.map((f) => readJson(path.join(runDir, 'scores', f)));
if (!raterFiles[sid].length) { console.error(`No rater output for ${sid}; run check.js --run first.`); process.exit(1); }
}
const nR = Math.min(...sids.map((s) => raterFiles[s].length));
const hypIds = prereg.ledger_hypotheses.map((h) => h.id);
const perRater = {};
for (const sid of sids) {
perRater[sid] = raterFiles[sid].slice(0, nR).map((r) => {
const f = raterFeatures(r, key[sid].duration_sec);
for (const h of hypIds) f[`hyp_${h}`] = r.hypotheses[h].score;
return f;
});
}
const raterFeatureIds = Object.keys(perRater[sids[0]][0]);
const values = {};
for (const fid of raterFeatureIds) values[fid] = sids.map((s) => mean(perRater[s].map((f) => f[fid])));
for (const fid of Object.keys(det[sids[0]]).filter((k) => k.startsWith('det_'))) values[fid] = sids.map((s) => det[s][fid]);
// ---------- pairwise craft ----------
const pairDir = path.join(runDir, 'pairs');
const pairs = (manifest.pairs || []).map((p) => ({ ...p, file: path.join(pairDir, `${p.id}.json`) })).filter((p) => fs.existsSync(p.file)).map((p) => ({ ...p, out: readJson(p.file) }));
const pairStats = { n_pairs: pairs.length, position_a_wins: null, cross_format_pairs: 0 };
if (pairs.length) {
const tally = (dimGetter, filter = () => true) => {
const w = Object.fromEntries(sids.map((s) => [s, 0])), g = Object.fromEntries(sids.map((s) => [s, 0]));
for (const p of pairs.filter(filter)) {
const win = dimGetter(p.out);
g[p.a]++; g[p.b]++;
if (win === 'A') w[p.a]++; else if (win === 'B') w[p.b]++; else { w[p.a] += 0.5; w[p.b] += 0.5; }
}
return sids.map((s) => winShare(w[s], g[s]));
};
values.pair_overall = tally((o) => o.overall.winner);
for (const d of DIMS) values[`pair_${d}`] = tally((o) => o.dims[d].winner);
const same = pairs.filter((p) => p.out.same_format);
pairStats.cross_format_pairs = pairs.length - same.length;
pairStats.same_format_pairs = same.length;
if (same.length) values.pair_overall_same_format = tally((o) => o.overall.winner, (p) => p.out.same_format);
const decided = pairs.filter((p) => p.out.overall.winner !== 'tie');
pairStats.position_a_wins = decided.length ? decided.filter((p) => p.out.overall.winner === 'A').length / decided.length : null;
const dimDecided = pairs.flatMap((p) => DIMS.map((d) => p.out.dims[d].winner)).filter((w) => w !== 'tie');
pairStats.position_a_wins_all_dims = dimDecided.length ? dimDecided.filter((w) => w === 'A').length / dimDecided.length : null;
pairStats.tie_rate = pairs.flatMap((p) => DIMS.map((d) => p.out.dims[d].winner)).filter((w) => w === 'tie').length / (pairs.length * DIMS.length);
}
// ---------- topic ----------
const topicPath = path.join(runDir, 'scores', 'topic.json');
const topicStats = {};
if (fs.existsSync(topicPath)) {
const tk = readJson(path.join(runDir, 'sealed', 'topic_key.json'));
const js = readJson(topicPath).judgments;
const w = Object.fromEntries(sids.map((s) => [s, 0])), g = Object.fromEntries(sids.map((s) => [s, 0]));
const byPair = {};
for (const j of js) {
const [x, y] = [tk[j.x], tk[j.y]];
g[x]++; g[y]++;
if (j.winner === 'tie') { w[x] += 0.5; w[y] += 0.5; } else w[tk[j.winner]]++;
const k = [x, y].sort().join('|');
(byPair[k] = byPair[k] || []).push(j.winner === 'tie' ? 'tie' : tk[j.winner]);
}
values.topic_pair = sids.map((s) => winShare(w[s], g[s]));
const both = Object.values(byPair).filter((v) => v.length === 2);
topicStats.order_consistency = both.length ? both.filter((v) => v[0] === v[1]).length / both.length : null;
topicStats.first_position_wins = js.filter((j) => j.winner === j.x).length / js.length;
topicStats.cards = readJson(path.join(runDir, 'blind', 'topic_cards.json')).map((c) => ({ ...c, sid: tk[c.tid] }));
}
// ---------- packaging, covariates ----------
const pkgPath = path.join(runDir, 'scores', 'packaging.json');
if (fs.existsSync(pkgPath)) {
const byPid = Object.fromEntries(readJson(pkgPath).titles.map((t) => [t.pid, t.score]));
values.pkg_title_score = sids.map((s) => byPid[key[s].pid]);
}
values.cov_duration_min = sids.map((s) => key[s].duration_sec / 60);
values.cov_age_days = sids.map((s) => key[s].age_days);
const covPath = path.join(runDir, 'covariates.json');
const postHocCovs = [];
if (fs.existsSync(covPath)) {
const cov = readJson(covPath);
for (const nm of new Set(Object.values(cov).flatMap((o) => Object.keys(o)))) {
values[`posthoc_${nm}`] = sids.map((s) => { const o = cov[key[s].video_id] || cov[s] || {}; return o[nm] ?? null; });
postHocCovs.push(`posthoc_${nm}`);
}
}
// ---------- specs ----------
const specs = {};
for (const f of prereg.features) specs[f.id] = { ...f, preregistered: true };
for (const h of prereg.ledger_hypotheses) specs[`hyp_${h.id}`] = { id: `hyp_${h.id}`, source: 'hyp', direction: h.direction, desc: h.statement, preregistered: true };
for (const fid of postHocCovs) specs[fid] = { id: fid, source: 'posthoc', direction: 0, desc: 'post-hoc covariate (added after unblinding)', preregistered: false };
if (values.pair_overall_same_format) specs.pair_overall_same_format = { id: 'pair_overall_same_format', source: 'pair', direction: 0, desc: 'pairwise overall win share using same-format pairs only', preregistered: false };
// ---------- performance ----------
const y = sids.map((s) => Math.log10(Math.max(key[s].perf, 1)));
const actualRank = ranks(y.map((v) => -v));
// ---------- reliability ----------
function reliability(fid) {
if (nR < 2 || !raterFeatureIds.includes(fid)) return null;
const rhos = [], agree = [];
for (let a = 0; a < nR; a++) for (let b = a + 1; b < nR; b++) {
const r = spearman(sids.map((s) => perRater[s][a][fid]), sids.map((s) => perRater[s][b][fid]));
if (r !== null) rhos.push(r);
}
const isBinary = fid.startsWith('item_');
if (isBinary) for (const s of sids) agree.push(new Set(perRater[s].map((f) => f[fid])).size === 1 ? 1 : 0);
const rho = rhos.length ? mean(rhos) : null;
const unanimous = isBinary ? mean(agree) : null;
// Rank agreement (rho) is meaningless when every video scores about the same, so a
// share-scale feature (0-1) also counts as reliable when raters agree in absolute terms.
const gaps = [];
for (const s of sids) for (let a = 0; a < nR; a++) for (let b = a + 1; b < nR; b++) gaps.push(Math.abs(perRater[s][a][fid] - perRater[s][b][fid]));
const gap = gaps.length ? mean(gaps) : null;
const shareScale = fid.startsWith('chk_') || ['ext_dependency_density', 'ext_filler_ratio'].includes(fid);
const ok = isBinary ? unanimous >= 0.67 : (rho === null || rho >= 0.4 || (shareScale && gap <= 0.1));
return { rater_rho: rho, mean_abs_gap: gap, unanimous_share: unanimous, ok };
}
// ---------- tests ----------
const tiers = [...new Set(sids.map((s) => key[s].bucket))];
const results = [];
for (const [fid, x] of Object.entries(values)) {
const spec = specs[fid] || { id: fid, source: fid.split('_')[0], direction: 0, desc: '(not in prereg; exploratory)', preregistered: false };
if (x.some((v) => v === null || v === undefined || Number.isNaN(v))) { results.push({ ...spec, verdict: 'missing', values: Object.fromEntries(sids.map((s, i) => [s, x[i]])) }); continue; }
const dir = spec.direction;
const t = permTest(x, y, dir);
const rel = reliability(fid);
const signed = t.rho === null ? null : (dir === 0 ? t.rho : dir * t.rho);
let looMin = null, looDriver = null;
if (t.rho !== null) sids.forEach((s, i) => {
const r = spearman(x.filter((_, k) => k !== i), y.filter((_, k) => k !== i));
if (r === null) return;
const sr = dir === 0 ? Math.abs(r) : dir * r;
if (looMin === null || sr < looMin) { looMin = sr; looDriver = s; }
});
let verdict;
if (t.rho === null) verdict = 'constant';
else if (['cov', 'posthoc', 'item'].includes(spec.source) || dir === 0) verdict = Math.abs(t.rho) >= 0.6 && t.p <= 0.1 ? 'exploratory_signal' : 'exploratory_none';
else if (signed >= 0.6 && t.p <= 0.10) verdict = 'explains';
else if (signed >= 0.3) verdict = 'weak_support';
else if (signed <= -0.3) verdict = 'contradicts';
else verdict = 'no_signal';
if (rel && !rel.ok && !verdict.startsWith('exploratory')) verdict += '_UNRELIABLE';
const fragile = verdict.startsWith('explains') && looMin !== null && looMin < 0.3;
results.push({
id: fid, source: spec.source, direction: dir, desc: spec.desc, preregistered: spec.preregistered,
rho: t.rho, signed_rho: signed, p: t.p, verdict, fragile, loo_min_signed_rho: looMin, loo_driver: fragile ? looDriver : null,
reliability: rel,
tier_means: Object.fromEntries(tiers.map((tr) => [tr, mean(x.filter((_, i) => key[sids[i]].bucket === tr))])),
values: Object.fromEntries(sids.map((s, i) => [s, x[i]])),
per_rater: raterFeatureIds.includes(fid) ? Object.fromEntries(sids.map((s) => [s, perRater[s].map((f) => f[fid])])) : undefined,
});
}
// ---------- confounds ----------
const confounds = results.filter((r) => ['cov', 'posthoc'].includes(r.source) && r.rho !== null && Math.abs(r.rho) >= 0.6);
for (const r of results) {
if (['cov', 'posthoc'].includes(r.source) || r.rho === null) continue;
const linked = confounds.filter((c) => Math.abs(spearman(values[r.id], values[c.id]) || 0) >= 0.6).map((c) => c.id);
if (linked.length) r.confounded_with = linked;
}
// ---------- predictions / stumped ----------
function predictionCheck(name, scores) {
const pr = ranks(scores.map((v) => -v));
const t = permTest(scores, y, 1);
return {
predictor: name, rho: t.rho, p: t.p,
per_sample: sids.map((s, i) => ({ sid: s, bucket: key[s].bucket, actual_rank: actualRank[i], predicted_rank: pr[i], miss: pr[i] - actualRank[i], stumped: Math.abs(pr[i] - actualRank[i]) >= Math.ceil(n / 3) })),
};
}
const predictors = ['chk_composite', 'pair_overall', 'topic_pair', 'hol_rater_percentile', 'pkg_title_score']
.filter((k) => values[k] && !values[k].some((v) => v === null)).map((k) => predictionCheck(k, values[k]));
const idea = values.topic_pair && values.pair_overall
? { craft_vs_topic_rho: spearman(values.pair_overall, values.topic_pair), topic_rho: spearman(values.topic_pair, y), craft_rho: spearman(values.pair_overall, y) }
: null;
const directional = results.filter((r) => r.preregistered && r.direction !== 0 && r.rho !== null);
const analysis = {
run_dir: runDir, n, metric: prereg.metric, prereg_hash_ok: hashOk, raters_per_sample: nR, rubric_hash: prereg.rubric_hash,
min_possible_p: 1 / [1, 1, 2, 6, 24, 120, 720, 5040, 40320][Math.min(n, 8)],
expected_false_explains: +(directional.length * 0.10).toFixed(1), n_directional_tests: directional.length,
key: Object.fromEntries(sids.map((s, i) => [s, { ...key[s], log10_perf: y[i], actual_rank: actualRank[i] }])),
pair_stats: pairStats, topic_stats: topicStats, idea_vs_execution: idea,
predictors, results, confounds: confounds.map((c) => c.id),
};
writeJson(path.join(runDir, 'analysis.json'), analysis);
// ---------- ledger ----------
if (!args['no-ledger'] && prereg.ledger_hypotheses.length && fs.existsSync(manifest.ledger)) {
const ledger = readJson(manifest.ledger);
for (const h of ledger.hypotheses) {
const r = results.find((x) => x.id === `hyp_${h.id}`);
if (!r) continue;
h.tests = h.tests || [];
if (h.tests.some((t) => t.run === runDir)) continue;
h.tests.push({ run: runDir, cohort: prereg.cohort, n, rho: r.rho, p: r.p, verdict: r.verdict, at: new Date().toISOString() });
const cohorts = (v) => new Set(h.tests.filter((t) => t.verdict === v).map((t) => t.cohort)).size;
if (cohorts('contradicts') >= 2) h.status = 'rejected';
else if (cohorts('explains') >= 2 && cohorts('contradicts') === 0) h.status = 'supported';
else h.status = 'testing';
}
writeJson(manifest.ledger, ledger);
}
// ---------- report-data.md ----------
const f2 = (v) => {
if (v === null || v === undefined) return '—';
if (typeof v !== 'number') return String(v).replace(/\|/g, '\\|');
if (v === 0) return '0';
if (Math.abs(v) >= 100) return v.toFixed(0);
if (Math.abs(v) < 0.01) return v.toPrecision(2);
return v.toFixed(2);
};
const L = [`# Cohort analysis data: ${path.basename(prereg.cohort)}`, ''];
L.push(`- n = ${n}, metric = log10(${prereg.metric}), raters per sample = ${nR}, rubric ${prereg.rubric_version || 'v1'} (${prereg.rubric_hash || '—'})`);
L.push(`- Pre-registration hash intact: **${hashOk ? 'yes' : 'NO, prereg.json was edited after prepare'}**`);
L.push(`- Smallest achievable p: ${f2(analysis.min_possible_p)}. ${directional.length} directional tests at p<=0.10 means about **${analysis.expected_false_explains} "explains" verdicts are expected by chance**.`);
if (pairs.length) L.push(`- Pairwise: ${pairs.length} pairs, ${pairStats.cross_format_pairs} judged cross-format. Position-A win rate (overall) ${f2(pairStats.position_a_wins)}, across all dims ${f2(pairStats.position_a_wins_all_dims)} (0.50 = no bias). Tie rate ${f2(pairStats.tie_rate)}.`);
if (topicStats.order_consistency !== undefined) L.push(`- Topic judge: both-order consistency ${f2(topicStats.order_consistency)}, first-position win rate ${f2(topicStats.first_position_wins)}.`);
if (idea) L.push(`- Idea vs execution: topic↔craft rho ${f2(idea.craft_vs_topic_rho)} (low = the two are separable); topic↔performance ${f2(idea.topic_rho)}; craft↔performance ${f2(idea.craft_rho)}.`);
L.push('', '## Unsealed key', '', '| Sample | Bucket | Title | Perf | Age (d) | Dur (min) | Topic card |', '|---|---|---|---|---|---|---|');
for (const s of sids) L.push(`| ${s} | ${key[s].bucket} | ${f2(key[s].title)} | ${f2(key[s].perf)} | ${key[s].age_days} | ${f2(key[s].duration_sec / 60)} | ${f2((topicStats.cards || []).find((c) => c.sid === s)?.card || '')} |`);
L.push('', '## Blind predictions vs actual', '');
for (const p of predictors) {
L.push(`**${p.predictor}**: rho = ${f2(p.rho)}, p = ${f2(p.p)}`, '', '| Sample | Bucket | Actual | Predicted | Stumped |', '|---|---|---|---|---|');
for (const r of [...p.per_sample].sort((a, b) => a.actual_rank - b.actual_rank)) L.push(`| ${r.sid} | ${r.bucket} | ${r.actual_rank} | ${r.predicted_rank} | ${r.stumped ? '**yes**' : ''} |`);
L.push('');
}
const order = ['explains', 'weak_support', 'no_signal', 'contradicts', 'constant', 'missing'];
const sortKey = (r) => { const k = order.indexOf(r.verdict.replace('_UNRELIABLE', '')); return (k < 0 ? 9 : k) + (r.verdict.endsWith('UNRELIABLE') ? 0.5 : 0); };
const relStr = (r) => (r.reliability ? (r.reliability.unanimous_share !== null ? `unanimous ${f2(r.reliability.unanimous_share)}` : `rho ${f2(r.reliability.rater_rho)}`) : (r.source === 'det' ? 'deterministic' : 'n/a'));
const tmStr = (r) => (r.tier_means ? Object.entries(r.tier_means).map(([k, v]) => `${k} ${f2(v)}`).join(', ') : '');
L.push('## Feature verdicts (pre-registered, directional)', '', '| Feature | Dir | rho | p | Verdict | Reliability | LOO min | Confounded with | Tier means |', '|---|---|---|---|---|---|---|---|---|');
for (const r of results.filter((r) => r.direction !== 0 && !['cov', 'posthoc', 'item'].includes(r.source)).sort((a, b) => sortKey(a) - sortKey(b))) {
L.push(`| ${r.id} | ${r.direction > 0 ? '+' : '−'} | ${f2(r.rho)} | ${f2(r.p)} | ${r.verdict}${r.fragile ? ` (fragile: ${r.loo_driver})` : ''} | ${relStr(r)} | ${f2(r.loo_min_signed_rho)} | ${(r.confounded_with || []).join(', ')} | ${tmStr(r)} |`);
}
L.push('', '## Exploratory, covariates and post-hoc (two-sided; never count as explanations)', '', '| Feature | rho | p | Verdict | Reliability | Tier means |', '|---|---|---|---|---|---|');
for (const r of results.filter((r) => (r.direction === 0 || ['cov', 'posthoc'].includes(r.source)) && r.source !== 'item')) L.push(`| ${r.id} | ${f2(r.rho)} | ${f2(r.p)} | ${r.verdict} | ${relStr(r)} | ${tmStr(r)} |`);
if (confounds.length) L.push('', `**Confound alert:** ${confounds.map((c) => c.id).join(', ')} correlate with performance at |rho| >= 0.6. Features moving with them are marked "confounded with".`);
fs.writeFileSync(path.join(runDir, 'report-data.md'), L.join('\n') + '\n', 'utf8');
// ---------- rule scorecard ----------
const sc = buildScorecard({ runDir, analysis, perRater, sids, calibrationDir: args.calibration ? path.resolve(args.calibration) : path.join(runDir, 'calibration') });
console.log(`Wrote analysis.json, report-data.md, rule-scorecard.md/json in ${runDir}`);
console.log(`Prereg intact: ${hashOk}. Rule status counts: ${JSON.stringify(sc.counts)}`);
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.