Docs/Skills and code/skill script-cohort-analysis

.claude/skills/script-cohort-analysis/scripts/scorecard.js

Code180 lines13.6 KBSkill: script-cohort-analysis

// Rule scorecard: for every rule (dimension, feature, checklist item) answer "did it work as
// expected, and what is the evidence?" Five independent lines of evidence:
//   1. reliability    - do independent blind raters agree?
//   2. discrimination - does it separate the videos at all, or is it constant / saturated?
//   3. calibration    - when we deliberately damage that property (perturb.js), does it move,
//                       and does it stay put when we damage something else?
//   4. convergence    - does the checklist score agree with the independent pairwise judges?
//   5. prediction     - does it predict performance in the pre-registered direction, and is
//                       that result robust (not fragile, not confounded)?
// Status = BROKEN_MEASUREMENT if 1-3 fail (then 5 is uninterpretable), otherwise from 5.
// Called by analyze.js; writes rule-scorecard.{md,json} and appends to the cross-run history.
const fs = require('fs');
const path = require('path');
const { readJson, writeJson, spearman } = require('./lib');
const { CHECKLIST, DIMS, ITEM_IDS, DIM_OF } = require('./checklist');

const f2 = (v) => (v === null || v === undefined || Number.isNaN(v) ? '—' : Math.abs(v) >= 100 ? v.toFixed(0) : v.toFixed(2));

function loadCalibration(dir) {
  const p = path.join(dir, 'calibration.json');
  return fs.existsSync(p) ? readJson(p) : null;
}

function calibrationFor(cal, fid) {
  if (!cal) return null;
  const asTarget = [], asGuard = [];
  for (const v of cal.variants) {
    const t = (v.targets || []).find((x) => x.feature === fid);
    if (t) asTarget.push({ kind: v.kind, base: v.base, delta: v.deltas[fid], ok: v.target_results[fid] });
    if ((v.guards || []).includes(fid)) asGuard.push({ kind: v.kind, base: v.base, delta: v.deltas[fid], ok: v.guard_results[fid] });
  }
  if (!asTarget.length && !asGuard.length) return null;
  return {
    target_detected: asTarget.filter((x) => x.ok).length, target_total: asTarget.length,
    guard_stable: asGuard.filter((x) => x.ok).length, guard_total: asGuard.length,
    detail: [...asTarget.map((x) => `${x.kind}/${x.base} target Δ${f2(x.delta)} ${x.ok ? '✓' : '✗'}`), ...asGuard.map((x) => `${x.kind}/${x.base} guard Δ${f2(x.delta)} ${x.ok ? '✓' : '✗'}`)],
  };
}

function statusOf(r, cal, discrimination, passRate, convergent) {
  const problems = [];
  if (convergent !== null && convergent < 0) problems.push(`disagrees with the pairwise judges (rho ${f2(convergent)})`);
  // ceiling: the checks pass for (almost) every script, so they cannot separate these videos,
  // even if they do catch gross damage in calibration
  if (passRate !== null && passRate >= 0.9 && discrimination < 0.25) return { status: 'SATURATED', problems: [`ceiling: ${Math.round(passRate * 100)}% of answers pass; spread across videos only ${f2(discrimination)}`, ...problems] };
  if (r.verdict === 'constant' || discrimination === 0) problems.push('constant across samples');
  if (r.reliability && !r.reliability.ok) problems.push(`raters disagree (${r.reliability.unanimous_share !== null ? `unanimous ${f2(r.reliability.unanimous_share)}` : `rho ${f2(r.reliability.rater_rho)}`})`);
  if (cal && cal.target_total && cal.target_detected / cal.target_total < 0.5) problems.push(`missed its own damage test (${cal.target_detected}/${cal.target_total})`);
  if (cal && cal.guard_total && cal.guard_stable / cal.guard_total < 0.5) problems.push(`moves when other properties are damaged (stable ${cal.guard_stable}/${cal.guard_total})`);
  if (problems.some((p) => !p.startsWith('disagrees with'))) return { status: 'BROKEN_MEASUREMENT', problems };
  const v = r.verdict.replace('_UNRELIABLE', '');
  if (r.direction === 0) return { status: v === 'exploratory_signal' ? 'EXPLORATORY_SIGNAL' : 'EXPLORATORY_NONE', problems };
  if (v === 'explains') return { status: r.fragile ? 'FRAGILE' : r.confounded_with ? 'CONFOUNDED' : 'WORKED', problems };
  if (v === 'weak_support') return { status: 'WEAK_SUPPORT', problems };
  if (v === 'contradicts') return { status: 'CONTRADICTED', problems };
  return { status: 'MEASURES_OK_NO_EFFECT', problems };
}

const MEANING = {
  WORKED: 'measured soundly and predicted performance in the expected direction, robustly',
  CONFOUNDED: 'predicted performance, but moves with a confound (age/format), so it could be the confound',
  FRAGILE: 'predicted performance only because of one video (drop it and the effect vanishes)',
  WEAK_SUPPORT: 'right direction, too weak for this n',
  MEASURES_OK_NO_EFFECT: 'measured soundly, but no relation to performance in this cohort',
  CONTRADICTED: 'measured soundly, and went the OPPOSITE way to the rule',
  BROKEN_MEASUREMENT: 'the measurement itself failed, so its relation to performance means nothing yet',
  SATURATED: 'almost every script passes these checks, so they cannot separate the videos (too easy for professional scripts)',
  EXPLORATORY_SIGNAL: 'not pre-registered with a direction; shows a signal worth registering',
  EXPLORATORY_NONE: 'not pre-registered with a direction; no signal',
};

function buildScorecard({ runDir, analysis, perRater, sids, calibrationDir }) {
  const cal = loadCalibration(calibrationDir);
  const byId = Object.fromEntries(analysis.results.map((r) => [r.id, r]));
  const key = analysis.key;
  const order = [...sids].sort((a, b) => key[a].actual_rank - key[b].actual_rank);
  const lbl = (s) => `${s}(${key[s].bucket[0]})`;
  const rules = [];

  const ruleFor = (fid, label, claim) => {
    const r = byId[fid];
    if (!r || r.verdict === 'missing') return null;
    const vals = order.map((s) => r.values[s]);
    const discrimination = Math.max(...vals) - Math.min(...vals);
    const c = calibrationFor(cal, fid);
    let convergent = null;
    if (fid.startsWith('chk_') && byId[`pair_${fid.slice(4)}`]) convergent = spearman(sids.map((s) => r.values[s]), sids.map((s) => byId[`pair_${fid.slice(4)}`].values[s]));
    const passRate = fid.startsWith('chk_') && r.per_rater ? (() => { const all = Object.values(r.per_rater).flat(); return all.reduce((a, b) => a + b, 0) / all.length; })() : null;
    const st = statusOf(r, c, discrimination, passRate, convergent);
    return {
      id: fid, rule: label, claim, direction: r.direction, status: st.status, meaning: MEANING[st.status], problems: st.problems,
      evidence: {
        reliability: r.reliability, discrimination, calibration: c, convergent_with_pairwise: convergent,
        prediction: { rho: r.rho, p: r.p, verdict: r.verdict, fragile: r.fragile, loo_driver: r.loo_driver, confounded_with: r.confounded_with || [] },
        values_best_to_worst: order.map((s) => [lbl(s), r.values[s]]),
      },
    };
  };

  for (const d of DIMS) rules.push(ruleFor(`chk_${d}`, d, `Scripts that pass more ${d} checks perform better`));
  rules.push(ruleFor('chk_composite', 'checklist composite', 'Scripts passing more checks overall perform better'));
  rules.push(ruleFor('pair_overall', 'pairwise overall craft', 'Scripts judged better head-to-head perform better'));
  rules.push(ruleFor('topic_pair', 'topic appeal (blind, topic cards only)', 'Videos on more appealing topics perform better'));
  for (const r of analysis.results.filter((x) => ['det', 'ext', 'hol', 'pkg', 'hyp'].includes(x.source))) rules.push(ruleFor(r.id, r.id, r.desc));
  const clean = rules.filter(Boolean);

  // checklist items
  const items = ITEM_IDS.map((id) => {
    const fid = `item_${id}`;
    const r = byId[fid];
    const passes = order.map((s) => {
      const arr = perRater[s].map((f) => f[fid]);
      return [lbl(s), `${arr.reduce((a, b) => a + b, 0)}/${arr.length}`];
    });
    const all = sids.flatMap((s) => perRater[s].map((f) => f[fid]));
    const dead = new Set(all).size === 1;
    return { id, dim: DIM_OF[id], text: CHECKLIST[DIM_OF[id]].find(([x]) => x === id)[1], passes, unanimous: r.reliability ? r.reliability.unanimous_share : null, dead, dead_value: dead ? all[0] : null, rho: r.rho };
  });

  const counts = clean.reduce((a, r) => { a[r.status] = (a[r.status] || 0) + 1; return a; }, {});

  // cross-run history
  // <cohort>/_runs/<run> -> history lives next to the cohort folders, shared across channels
  const histPath = path.join(path.dirname(path.dirname(runDir)), '..', '_rule_history.json');
  const hist = fs.existsSync(histPath) ? readJson(histPath) : { runs: [] };
  hist.runs = hist.runs.filter((x) => x.run !== runDir);
  hist.runs.push({ run: runDir, cohort: path.basename(path.dirname(path.dirname(runDir))), rubric_hash: analysis.rubric_hash, at: new Date().toISOString(), n: analysis.n, rules: Object.fromEntries(clean.map((r) => [r.id, { status: r.status, rho: r.evidence.prediction.rho }])) });
  writeJson(histPath, hist);
  const historyOf = (id) => {
    const c = {};
    for (const run of hist.runs) if (run.rules[id]) c[run.rules[id].status] = (c[run.rules[id].status] || 0) + 1;
    return Object.entries(c).map(([k, v]) => `${k}×${v}`).join(', ');
  };

  const out = { run: runDir, rubric_hash: analysis.rubric_hash, calibration: cal ? { dir: calibrationDir, rubric_hash: cal.rubric_hash, passed: cal.passed, total: cal.total } : null, counts, pair_stats: analysis.pair_stats, topic_stats: { order_consistency: analysis.topic_stats.order_consistency, first_position_wins: analysis.topic_stats.first_position_wins }, rules: clean, items, history_file: histPath };
  writeJson(path.join(runDir, 'rule-scorecard.json'), out);

  // markdown
  const L = ['# Rule scorecard', '',
    'For each rule: **did it work as expected, and what is the evidence?**',
    `Rubric ${analysis.rubric_hash || '—'}, n = ${analysis.n}, ${analysis.raters_per_sample} raters per video. About ${analysis.expected_false_explains} chance "explains" verdicts are expected across ${analysis.n_directional_tests} tests.`,
    cal ? `Calibration (damage test): ${cal.passed}/${cal.total} manipulations detected cleanly. Source: ${path.relative(runDir, path.join(calibrationDir, 'calibration.json')) || calibrationDir}${cal.rubric_hash && cal.rubric_hash !== analysis.rubric_hash ? ' (**WARNING: calibrated on a different rubric version**)' : ''}.` : 'Calibration: **not run for this rubric**, so line 3 of the evidence is missing (run perturb.js).',
    '', '## Status counts', '', ...Object.entries(counts).map(([k, v]) => `- **${k}** (${v}): ${MEANING[k]}`), ''];

  const ps = analysis.pair_stats || {};
  const ts = analysis.topic_stats || {};
  L.push('## Health of the comparison methods', '',
    `- Pairwise craft: ${ps.n_pairs || 0} pairs; ${ps.cross_format_pairs ?? '—'} judged cross-format; position-A win rate ${f2(ps.position_a_wins)} overall, ${f2(ps.position_a_wins_all_dims)} across dims (0.50 = no order bias); tie rate ${f2(ps.tie_rate)}.`,
    `- Topic judge: both-order consistency ${f2(ts.order_consistency)} (1.00 = never contradicts itself); first-position win rate ${f2(ts.first_position_wins)}.`,
    analysis.idea_vs_execution ? `- Idea vs execution separable? topic↔craft rho ${f2(analysis.idea_vs_execution.craft_vs_topic_rho)} (near 0 = measuring different things).` : '', '');

  L.push('## Rules', '', '| Rule | Expected | Status | Reliability | Discrimination | Calibration | Agrees w/ pairwise | Prediction (rho, p) | History | Problems / notes |', '|---|---|---|---|---|---|---|---|---|---|');
  const rank = ['WORKED', 'CONFOUNDED', 'FRAGILE', 'WEAK_SUPPORT', 'MEASURES_OK_NO_EFFECT', 'CONTRADICTED', 'SATURATED', 'BROKEN_MEASUREMENT', 'EXPLORATORY_SIGNAL', 'EXPLORATORY_NONE'];
  for (const r of [...clean].sort((a, b) => rank.indexOf(a.status) - rank.indexOf(b.status))) {
    const e = r.evidence;
    const rel = e.reliability ? (e.reliability.unanimous_share !== null ? `unan ${f2(e.reliability.unanimous_share)}` : `rho ${f2(e.reliability.rater_rho)} gap ${f2(e.reliability.mean_abs_gap)}`) : 'det';
    const calS = e.calibration ? `${e.calibration.target_total ? `hit ${e.calibration.target_detected}/${e.calibration.target_total}` : ''}${e.calibration.guard_total ? ` stable ${e.calibration.guard_stable}/${e.calibration.guard_total}` : ''}`.trim() : '—';
    const notes = [...r.problems, ...(e.prediction.confounded_with.length ? [`moves with ${e.prediction.confounded_with.join(', ')}`] : []), ...(e.prediction.fragile ? [`driven by ${e.prediction.loo_driver}`] : [])].join('; ');
    L.push(`| ${r.rule} | ${r.direction > 0 ? 'higher = better' : r.direction < 0 ? 'lower = better' : 'exploratory'} | **${r.status}** | ${rel} | ${f2(e.discrimination)} | ${calS} | ${f2(e.convergent_with_pairwise)} | ${f2(e.prediction.rho)}, ${f2(e.prediction.p)} | ${historyOf(r.id) || '—'} | ${notes} |`);
  }

  L.push('', '## Evidence by dimension (values ordered best → worst performer)', '');
  for (const d of DIMS) {
    const r = clean.find((x) => x.id === `chk_${d}`);
    if (!r) continue;
    L.push(`### ${d}: ${r.status}`, '', `Values: ${r.evidence.values_best_to_worst.map(([s, v]) => `${s} ${f2(v)}`).join(' · ')}`);
    if (r.evidence.calibration) L.push(`Damage test: ${r.evidence.calibration.detail.join('; ')}`);
    L.push('', '| Check | Pass count per video (best → worst) | Unanimous | Notes |', '|---|---|---|---|');
    for (const it of items.filter((x) => x.dim === d)) {
      L.push(`| **${it.id}** ${it.text} | ${it.passes.map(([s, v]) => `${s} ${v}`).join(', ')} | ${f2(it.unanimous)} | ${it.dead ? `dead item: always ${it.dead_value ? 'pass' : 'fail'}` : ''} |`);
    }
    L.push('');
  }
  L.push(`Rater files with per-check quotes: \`scores/S*.r*.json\` → \`checks.<ID>\`. Pairwise reasons: \`pairs/P*.json\`. Cross-run history: \`${histPath}\`.`);
  fs.writeFileSync(path.join(runDir, 'rule-scorecard.md'), L.filter((x) => x !== null).join('\n') + '\n', 'utf8');
  return out;
}

module.exports = { buildScorecard };

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.