Docs/Skills and code/skill script-cohort-analysis

.claude/skills/script-cohort-analysis/scripts/prepare.js

Code298 lines13.8 KBSkill: script-cohort-analysis

#!/usr/bin/env node
// Phase 1: blind the cohort, compute deterministic features, freeze the pre-registration,
// and write one task file per blind rater.
//
//   node prepare.js --cohort <dir-of-transcript-txts> [--out <runDir>] [--raters 2]
//                   [--seed N] [--ledger <path>] [--metric views_per_day|views|performance]

const fs = require('fs');
const path = require('path');
const { readJson, writeJson, sha256, parseArgs, toSec, rng, shuffle, mean, sd } = require('./lib');
const { FEATURES } = require('./features');
const { renderMarkdown } = require('./checklist');

const SKILL_ROOT = path.resolve(__dirname, '..');
const args = parseArgs(process.argv.slice(2));
if (!args.cohort) {
  console.error('usage: node prepare.js --cohort <dir> [--out <runDir>] [--raters 2] [--seed N] [--ledger <path>] [--metric views_per_day|views]');
  process.exit(1);
}
const cohortDir = path.resolve(args.cohort);
const nRaters = Number(args.raters || 3);
const audience = args.audience || 'viewers of a Hindi-language explainer YouTube channel in India';
const seed = Number(args.seed || Date.now() % 2147483647);
const metric = args.metric || 'views_per_day';
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+/, '').replace('T', '-');
const runDir = path.resolve(args.out || path.join(cohortDir, '_runs', stamp));
const ledgerPath = path.resolve(args.ledger || path.join(cohortDir, '..', '_script_ledger.json'));
const lexicon = readJson(path.join(__dirname, 'lexicon.json'));

// ---------- parse transcript files ----------
function parseNum(s) {
  if (!s) return null;
  const m = String(s).replace(/,/g, '').match(/-?\d+(\.\d+)?/);
  return m ? Number(m[0]) : null;
}
function field(text, label) {
  const m = text.match(new RegExp('^' + label + ':\\s*(.*)$', 'm'));
  return m ? m[1].trim() : null;
}
function parseFile(file) {
  const raw = fs.readFileSync(file, 'utf8');
  const tIdx = raw.indexOf('=== TRANSCRIPT ===');
  if (tIdx < 0) throw new Error(`${file}: no "=== TRANSCRIPT ===" section`);
  const header = raw.slice(0, tIdx);
  const body = raw.slice(tIdx);
  const chunks = [];
  for (const line of body.split(/\r?\n/)) {
    const m = line.match(/^\[(\d{1,2}(?::\d{2}){1,2})\]\s*(.*)$/);
    if (m) chunks.push({ ts: m[1], start: toSec(m[1]), text: m[2] });
  }
  const durMatch = (field(header, 'Duration') || '').match(/(?:(\d+)h\s*)?(?:(\d+)m\s*)?(\d+)s/);
  const duration = durMatch
    ? (Number(durMatch[1] || 0) * 3600 + Number(durMatch[2] || 0) * 60 + Number(durMatch[3] || 0))
    : (chunks.length ? chunks[chunks.length - 1].start + 30 : null);
  const ageMatch = (field(header, 'Published') || '').match(/age (\d+) days/);
  const langLine = field(body, 'Language');
  return {
    file: path.basename(file),
    title: field(header, 'Title'),
    video_id: field(header, 'Video ID'),
    bucket: (field(header, 'Sample bucket') || '').split(/\s+/)[0].toLowerCase() || null,
    views: parseNum(field(header, 'Views')),
    views_per_day: parseNum(field(header, 'Views per day')),
    performance_score: parseNum(field(header, 'Performance score')),
    rank_reported: parseNum(field(header, 'Rank by views/day')),
    age_days: ageMatch ? Number(ageMatch[1]) : null,
    duration_sec: duration,
    language: langLine ? langLine.split('[')[0].trim() : 'unknown',
    chunks,
  };
}

// ---------- deterministic features ----------
function tokenize(text) {
  return text.split(/[\s।?!,;:"“”‘’()\[\]…|]+/)
    .map((t) => t.replace(/\.+$/, '').toLowerCase())
    .filter(Boolean);
}
function countTerms(tokens, terms) {
  let n = 0;
  const seqs = terms.map((t) => tokenize(t));
  for (let i = 0; i < tokens.length; i++) {
    for (const s of seqs) {
      if (s.length && s.every((w, k) => tokens[i + k] === w)) n++;
    }
  }
  return n;
}
function sentences(text) {
  return text.split(/(?<=[।?!])|\.(?=\s|$)/).map((s) => s.trim()).filter((s) => tokenize(s).length > 0);
}
function isQuestion(sentence, qset) {
  if (sentence.trim().endsWith('?')) return true;
  return tokenize(sentence).some((t) => qset.has(t));
}
function numberTokens(tokens, magnitude) {
  const mag = new Set(magnitude.map((t) => t.toLowerCase()));
  return tokens.filter((t) => /\d/.test(t) || t.includes('₹') || mag.has(t)).length;
}
function detFeatures(doc) {
  const full = doc.chunks.map((c) => c.text).join(' ');
  const hook = doc.chunks.filter((c) => c.start < 60).map((c) => c.text).join(' ');
  const tokens = tokenize(full);
  const hookTokens = tokenize(hook);
  const W = tokens.length || 1;
  const qset = new Set(lexicon.question_words.map((t) => t.toLowerCase()));
  const sents = sentences(full);
  const lens = sents.map((s) => tokenize(s).length);
  const first = tokens.slice(0, 1500);
  const per100 = (n) => (100 * n) / W;
  return {
    words: tokens.length,
    sentences: sents.length,
    det_question_rate: per100(sents.filter((s) => isQuestion(s, qset)).length),
    det_second_person_rate: per100(countTerms(tokens, lexicon.second_person)),
    det_number_rate: per100(numberTokens(tokens, lexicon.magnitude)),
    det_contrast_rate: per100(countTerms(tokens, lexicon.contrast)),
    det_causal_rate: per100(countTerms(tokens, lexicon.causal)),
    det_additive_rate: per100(countTerms(tokens, lexicon.additive)),
    det_hook_questions: sentences(hook).filter((s) => isQuestion(s, qset)).length,
    det_hook_numbers: numberTokens(hookTokens, lexicon.magnitude),
    det_imagine_rate: (1000 * countTerms(tokens, lexicon.imagine)) / W,
    det_sentence_len_cv: lens.length > 1 ? sd(lens) / mean(lens) : 0,
    det_wpm: doc.duration_sec ? tokens.length / (doc.duration_sec / 60) : null,
    det_ttr: first.length ? new Set(first).size / first.length : null,
  };
}

// ---------- main ----------
const files = fs.readdirSync(cohortDir)
  .filter((f) => f.endsWith('.txt'))
  .map((f) => path.join(cohortDir, f));
if (files.length < 4) {
  console.error(`Need at least 4 transcripts in ${cohortDir}, found ${files.length}.`);
  process.exit(1);
}
const docs = files.map(parseFile);
for (const d of docs) {
  if (d.views_per_day === null && d.views !== null && d.age_days) d.views_per_day = d.views / Math.max(d.age_days, 30);
  // 'performance' = a cross-channel blend (views vs the channel's own norm, views/day, like rate) for cohorts that mix channels.
  d.perf = metric === 'views' ? d.views : metric === 'performance' ? d.performance_score : d.views_per_day;
  if (d.perf === null) { console.error(`${d.file}: no ${metric} found`); process.exit(1); }
}

const rand = rng(seed);
const order = shuffle(docs, rand);
const titleOrder = shuffle(order.map((_, i) => i), rand);
const pidOf = {};
titleOrder.forEach((idx, k) => { pidOf[idx] = `P${k + 1}`; });

fs.mkdirSync(path.join(runDir, 'blind'), { recursive: true });
fs.mkdirSync(path.join(runDir, 'scores'), { recursive: true });
fs.mkdirSync(path.join(runDir, 'tasks'), { recursive: true });

const key = {};
const features = {};
order.forEach((d, i) => {
  const sid = `S${i + 1}`;
  const mins = Math.floor(d.duration_sec / 60), secs = d.duration_sec % 60;
  const blind = [
    `BLIND SAMPLE ${sid}`,
    `Duration: ${mins}m ${String(secs).padStart(2, '0')}s | Transcript language: ${d.language} | Chunks are ~30s, timestamps mark chunk starts`,
    '',
    ...d.chunks.map((c) => `[${c.ts}] ${c.text}`),
    '',
  ].join('\n');
  fs.writeFileSync(path.join(runDir, 'blind', `${sid}.txt`), blind, 'utf8');
  features[sid] = { duration_sec: d.duration_sec, ...detFeatures(d) };
  key[sid] = {
    file: d.file, video_id: d.video_id, title: d.title, bucket: d.bucket,
    views: d.views, views_per_day: d.views_per_day, age_days: d.age_days,
    duration_sec: d.duration_sec, perf: d.perf, pid: pidOf[i],
  };
});
writeJson(path.join(runDir, 'blind', 'titles.json'),
  titleOrder.map((idx, k) => ({ pid: `P${k + 1}`, title: order[idx].title })));
writeJson(path.join(runDir, 'features.json'), features);
writeJson(path.join(runDir, 'sealed', 'key.json'), key);


// ledger hypotheses become extra rater questions
let ledger = { hypotheses: [] };
if (fs.existsSync(ledgerPath)) ledger = readJson(ledgerPath);
const testable = ledger.hypotheses.filter((h) => ['untested', 'testing'].includes(h.status) && h.rater_question);

// the exact rubric this run uses is frozen into the run directory
const checklistMd = renderMarkdown();
fs.writeFileSync(path.join(runDir, 'checklist.md'), checklistMd, 'utf8');
const ref = (f) => path.join(SKILL_ROOT, 'references', f);
const rubricHash = sha256([checklistMd, ...['rubric.md', 'rater-prompt.md', 'pair-prompt.md', 'topic-prompt.md']
  .map((f) => fs.readFileSync(ref(f), 'utf8'))].join('\n')).slice(0, 16);

const prereg = {
  created_at: new Date().toISOString(),
  cohort: cohortDir,
  n: docs.length,
  metric,
  performance_transform: 'log10',
  raters_per_sample: nRaters,
  rubric_version: 'v3-checklist',
  rubric_hash: rubricHash,
  audience,
  seed,
  features: FEATURES,
  ledger_hypotheses: testable.map((h) => ({
    id: h.id, statement: h.statement, rater_question: h.rater_question, direction: h.direction ?? 1,
  })),
  decision_rules: {
    explains: 'signed rho >= 0.6 AND one-sided permutation p <= 0.10 AND measurement sound (direction != 0)',
    weak_support: '0.3 <= signed rho < 0.6, or explains-level rho with p > 0.10',
    no_signal: '|rho| < 0.3',
    contradicts: 'signed rho <= -0.3',
    unreliable: 'mean inter-rater spearman < 0.4 (overrides above)',
    exploratory: 'direction 0: report rho and two-sided p only',
    stumped: 'sample whose composite-predicted rank differs from actual rank by >= ceil(n/3)',
    rule_status: 'see scorecard.js: WORKED / CONFOUNDED / MEASURES_OK_NO_EFFECT / CONTRADICTED / BROKEN_MEASUREMENT',
  },
};
const preregText = JSON.stringify(prereg, null, 2);
writeJson(path.join(runDir, 'prereg.json'), prereg);
fs.writeFileSync(path.join(runDir, 'prereg.sha256'), sha256(preregText) + '\n');

// ---------- task files ----------
const BLIND_RULE = 'Read ONLY the files listed above. Do not list directories, glob, grep or open any other file: other files reveal the answers and would invalidate the experiment.';
const validate = (out) => `node "${path.join(__dirname, 'check.js')}" --file "${out}"  (fix and rewrite until it prints OK)`;
const hypBlock = testable.length
  ? '\nExtra hypothesis questions (answer each in a `hypotheses` object as {"score": 1-5, "ts": "", "quote": "", "gloss": ""}):\n' +
    testable.map((h) => `- ${h.id}: ${h.rater_question}`).join('\n') + '\n'
  : '\nNo extra hypothesis questions this run (`hypotheses` = {}).\n';
const sids = Object.keys(key);

const tasks = [];
for (const sid of sids) {
  for (let r = 1; r <= nRaters; r++) {
    const out = path.join(runDir, 'scores', `${sid}.r${r}.json`);
    const p = path.join(runDir, 'tasks', `rate_${sid}_r${r}.md`);
    fs.writeFileSync(p, [
      `# Blind rating task: ${sid}, rater ${r}`, '',
      `1. Read the rater instructions: ${ref('rater-prompt.md')}`,
      `2. Read the definitions: ${ref('rubric.md')}`,
      `3. Read the checklist: ${path.join(runDir, 'checklist.md')}`,
      `4. Read the transcript: ${path.join(runDir, 'blind', `${sid}.txt`)}`,
      `5. Write your JSON to: ${out}`,
      `6. Validate: ${validate(out)}`,
      hypBlock, BLIND_RULE, '',
    ].join('\n'), 'utf8');
    tasks.push(p);
  }
}

// all n-choose-2 pairs, positions balanced so each sample is "A" about half the time
const pairs = [];
const asA = Object.fromEntries(sids.map((s) => [s, 0]));
for (const [i, j] of shuffle(sids.flatMap((s, i) => sids.slice(i + 1).map((_, k) => [i, i + 1 + k])), rand)) {
  const [x, y] = [sids[i], sids[j]];
  const [a, b] = asA[x] <= asA[y] ? [x, y] : [y, x];
  asA[a]++;
  pairs.push({ id: `P${String(pairs.length + 1).padStart(2, '0')}`, a, b });
}
const pairTasks = pairs.map((pr) => {
  const out = path.join(runDir, 'pairs', `${pr.id}.json`);
  const p = path.join(runDir, 'tasks', `pair_${pr.id}.md`);
  fs.writeFileSync(p, [
    `# Blind pairwise comparison ${pr.id}`, '',
    `1. Read the instructions: ${ref('pair-prompt.md')}`,
    `2. Read transcript A (${pr.a}): ${path.join(runDir, 'blind', `${pr.a}.txt`)}`,
    `3. Read transcript B (${pr.b}): ${path.join(runDir, 'blind', `${pr.b}.txt`)}`,
    `4. Write your JSON (pair_id "${pr.id}", a "${pr.a}", b "${pr.b}") to: ${out}`,
    `5. Validate: ${validate(out)}`, '', BLIND_RULE, '',
  ].join('\n'), 'utf8');
  return p;
});
fs.mkdirSync(path.join(runDir, 'pairs'), { recursive: true });

const pkgOut = path.join(runDir, 'scores', 'packaging.json');
const pkgTask = path.join(runDir, 'tasks', 'packaging.md');
fs.writeFileSync(pkgTask, [
  '# Blind packaging task', '',
  `1. Read the instructions: ${ref('packaging-prompt.md')}`,
  `2. Read the titles: ${path.join(runDir, 'blind', 'titles.json')}`,
  `3. Write your JSON to: ${pkgOut}`,
  `4. Validate: ${validate(pkgOut)}`, '', BLIND_RULE, '',
].join('\n'), 'utf8');

writeJson(path.join(runDir, 'manifest.json'), {
  run_dir: runDir, cohort: cohortDir, ledger: ledgerPath, samples: sids, raters: nRaters,
  rubric_hash: rubricHash, audience, pairs,
  rater_tasks: tasks, pair_tasks: pairTasks, packaging_task: pkgTask,
});

console.log(`Run directory: ${runDir}`);
console.log(`Samples: ${sids.join(', ')}  (key sealed in sealed/key.json — do not open before analyze)`);
console.log(`Pre-registration hash: ${sha256(preregText).slice(0, 16)} | rubric hash: ${rubricHash}`);
console.log(`Ledger hypotheses under test: ${testable.length}`);
console.log(`\nWave 1 (run in parallel): ${tasks.length} rater tasks + ${pairTasks.length} pair tasks + 1 packaging task`);
[...tasks, ...pairTasks, pkgTask].forEach((t) => console.log('  ' + t));
console.log(`\nWave 2 (after wave 1): node "${path.join(__dirname, 'topics.js')}" --run "${runDir}"  -> writes the topic task`);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.