Docs/Skills and code/skill script-cohort-analysis

.claude/skills/script-cohort-analysis/scripts/check.js

Code159 lines8.3 KBSkill: script-cohort-analysis

#!/usr/bin/env node
// Validate one output file, or every expected output of a run.
//   node check.js --file <runDir>/scores/S1.r1.json | <runDir>/pairs/P01.json | <runDir>/scores/topic.json | packaging.json
//   node check.js --run <runDir>
const fs = require('fs');
const path = require('path');
const { readJson, parseArgs, toSec } = require('./lib');
const { DIMS, ITEM_IDS } = require('./checklist');

const FUNCTIONS = ['hook', 'promise', 'context', 'value', 'example', 'story', 'complication', 'payoff',
  'recap', 'transition', 'cta', 'sponsor', 'filler', 'repetition'];
const isInt15 = (v) => Number.isInteger(v) && v >= 1 && v <= 5;

function runDirOf(file) {
  // outputs live in <run>/scores or <run>/pairs; calibration runs nest the same layout
  return path.resolve(path.dirname(file), '..');
}
function blindDuration(runDir, sid) {
  const p = path.join(runDir, 'blind', `${sid}.txt`);
  if (!fs.existsSync(p)) return null;
  const m = fs.readFileSync(p, 'utf8').match(/Duration: (\d+)m (\d+)s/);
  return m ? Number(m[1]) * 60 + Number(m[2]) : null;
}

function checkRater(obj, file, runDir) {
  const errs = [];
  const sid = path.basename(file).split('.')[0];
  if (obj.sample_id !== sid) errs.push(`sample_id should be "${sid}"`);
  for (const k of ['topic_card', 'promise', 'sections', 'loops', 'checks', 'holistic']) if (obj[k] === undefined) errs.push(`missing "${k}"`);
  if (errs.length) return errs;
  const words = String(obj.topic_card).trim().split(/\s+/).filter(Boolean).length;
  if (!words || words > 35) errs.push(`topic_card must be 1-25 words (got ${words})`);
  if (toSec(obj.promise.timestamp) === null) errs.push('promise.timestamp invalid');

  const dur = blindDuration(runDir, sid);
  if (!Array.isArray(obj.sections) || obj.sections.length < 3) errs.push('sections: need at least 3');
  else {
    const ids = obj.sections.map((s) => s.id);
    let prevEnd = null;
    obj.sections.forEach((s, i) => {
      const a = toSec(s.start), b = toSec(s.end);
      if (!s.id) errs.push(`sections[${i}].id missing`);
      if (a === null || b === null || b <= a) errs.push(`sections[${i}]: bad start/end "${s.start}"-"${s.end}"`);
      if (!FUNCTIONS.includes(s.function)) errs.push(`sections[${i}].function "${s.function}" not in ${FUNCTIONS.join('|')}`);
      if (!isInt15(s.interest)) errs.push(`sections[${i}].interest must be integer 1-5`);
      if (!Array.isArray(s.depends_on)) errs.push(`sections[${i}].depends_on must be an array of section ids`);
      else s.depends_on.forEach((d) => { if (!ids.includes(d)) errs.push(`sections[${i}].depends_on has unknown id "${d}"`); });
      if (!Array.isArray(s.references_unintroduced)) errs.push(`sections[${i}].references_unintroduced must be an array`);
      if (typeof s.summary !== 'string' || !s.summary.trim()) errs.push(`sections[${i}].summary missing`);
      if (i === 0 && a > 30) errs.push('sections must start at 00:00');
      if (prevEnd !== null && a !== null && Math.abs(a - prevEnd) > 31) errs.push(`sections[${i}] leaves a gap/overlap of ${a - prevEnd}s`);
      prevEnd = b;
    });
    if (dur && prevEnd !== null && prevEnd < dur - 90) errs.push(`sections end at ${prevEnd}s but video is ${dur}s; tile the full runtime`);
  }
  if (!Array.isArray(obj.loops)) errs.push('loops must be an array');
  else obj.loops.forEach((l, i) => {
    if (toSec(l.opened_at) === null) errs.push(`loops[${i}].opened_at invalid`);
    if (l.closed_at !== null && toSec(l.closed_at) === null) errs.push(`loops[${i}].closed_at must be timestamp or null`);
    if (!['major', 'minor'].includes(l.size)) errs.push(`loops[${i}].size must be major|minor`);
  });
  for (const id of ITEM_IDS) {
    const c = obj.checks[id];
    if (!c) { errs.push(`checks.${id} missing`); continue; }
    if (typeof c.pass !== 'boolean') errs.push(`checks.${id}.pass must be true or false`);
  }
  const extra = Object.keys(obj.checks).filter((k) => !ITEM_IDS.includes(k));
  if (extra.length) errs.push(`checks has unknown ids: ${extra.join(', ')}`);
  const prereg = path.join(runDir, 'prereg.json');
  if (fs.existsSync(prereg)) {
    for (const h of readJson(prereg).ledger_hypotheses || []) {
      const v = obj.hypotheses && obj.hypotheses[h.id];
      if (!v || !isInt15(v.score)) errs.push(`hypotheses.${h.id}.score (1-5) required`);
    }
  }
  const p = obj.holistic.percentile;
  if (typeof p !== 'number' || p < 0 || p > 100) errs.push('holistic.percentile must be 0-100');
  return errs;
}

function checkPair(obj, file, runDir) {
  const errs = [];
  const id = path.basename(file, '.json');
  const man = readJson(path.join(runDir, 'manifest.json'));
  const pr = (man.pairs || []).find((p) => p.id === id);
  if (!pr) return [`${id} is not a pair in manifest.json`];
  if (obj.pair_id !== id || obj.a !== pr.a || obj.b !== pr.b) errs.push(`pair_id/a/b must be "${id}"/"${pr.a}"/"${pr.b}"`);
  const W = ['A', 'B', 'tie'];
  for (const d of DIMS) {
    const v = obj.dims && obj.dims[d];
    if (!v || !W.includes(v.winner)) errs.push(`dims.${d}.winner must be A|B|tie`);
  }
  if (!obj.overall || !W.includes(obj.overall.winner)) errs.push('overall.winner must be A|B|tie');
  if (typeof obj.same_format !== 'boolean') errs.push('same_format must be true or false');
  return errs;
}

function checkTopic(obj, runDir) {
  const man = readJson(path.join(runDir, 'manifest.json'));
  const want = man.topic_comparisons || [];
  const got = obj.judgments || [];
  const errs = [];
  if (got.length !== want.length) errs.push(`judgments: expected ${want.length}, got ${got.length}`);
  want.forEach(([x, y], k) => {
    const j = got[k];
    if (!j || j.x !== x || j.y !== y) errs.push(`judgments[${k}] must be {x:"${x}", y:"${y}"}`);
    else if (![x, y, 'tie'].includes(j.winner)) errs.push(`judgments[${k}].winner must be "${x}", "${y}" or "tie"`);
  });
  return errs.slice(0, 10);
}

function checkPackaging(obj, runDir) {
  const errs = [];
  const titles = readJson(path.join(runDir, 'blind', 'titles.json')).map((t) => t.pid);
  const got = (obj.titles || []).map((t) => t.pid);
  for (const pid of titles) if (!got.includes(pid)) errs.push(`titles: missing ${pid}`);
  (obj.titles || []).forEach((t) => { if (!isInt15(t.score)) errs.push(`${t.pid}.score must be 1-5`); });
  if (!Array.isArray(obj.ranking) || [...obj.ranking].sort().join() !== [...titles].sort().join()) errs.push('ranking must list every pid exactly once');
  return errs;
}

function checkFile(file) {
  const runDir = runDirOf(file);
  let obj;
  try { obj = readJson(file); } catch (e) { return [`not valid JSON: ${e.message}`]; }
  const base = path.basename(file);
  if (base === 'packaging.json') return checkPackaging(obj, runDir);
  if (base === 'topic.json') return checkTopic(obj, runDir);
  if (/^P\d+\.json$/.test(base)) return checkPair(obj, file, runDir);
  if (/^S\d+\.r\d+\.json$/.test(base)) return checkRater(obj, file, runDir);
  return [`unrecognised output file name ${base}`];
}

const args = parseArgs(process.argv.slice(2));
if (args.file) {
  const errs = checkFile(path.resolve(args.file));
  if (errs.length) { console.log('ERRORS:\n- ' + errs.join('\n- ')); process.exit(1); }
  console.log('OK');
} else if (args.run) {
  const runDir = path.resolve(args.run);
  const man = readJson(path.join(runDir, 'manifest.json'));
  const expected = [];
  for (const sid of man.samples) for (let r = 1; r <= man.raters; r++) expected.push(path.join(runDir, 'scores', `${sid}.r${r}.json`));
  for (const p of man.pairs || []) expected.push(path.join(runDir, 'pairs', `${p.id}.json`));
  if (man.packaging_task) expected.push(path.join(runDir, 'scores', 'packaging.json'));
  if (man.topic_task) expected.push(path.join(runDir, 'scores', 'topic.json'));
  let bad = 0;
  for (const f of expected) {
    if (!fs.existsSync(f)) { console.log(`MISSING ${path.relative(runDir, f)}`); bad++; continue; }
    const errs = checkFile(f);
    if (errs.length) { console.log(`INVALID ${path.relative(runDir, f)}: ${errs.join('; ')}`); bad++; }
  }
  console.log(`${expected.length - bad}/${expected.length} outputs valid${man.topic_task ? '' : ' (topic task not created yet: run topics.js after wave 1)'}`);
  process.exit(bad ? 1 : 0);
} else {
  console.error('usage: node check.js --file <output.json> | --run <runDir>');
  process.exit(1);
}

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.