Docs/Skills and code/skill script-cohort-analysis

.claude/skills/script-cohort-analysis/scripts/topics.js

Code56 lines3.3 KBSkill: script-cohort-analysis

#!/usr/bin/env node
// Wave 2: build blind topic cards from the raters' neutral topic_card fields and write the
// topic-comparison task. The topic judge never sees a transcript, so execution cannot leak in.
//   node topics.js --run <runDir> [--rater r1]
const fs = require('fs');
const path = require('path');
const { readJson, writeJson, parseArgs, rng, shuffle } = require('./lib');

const SKILL_ROOT = path.resolve(__dirname, '..');
const args = parseArgs(process.argv.slice(2));
if (!args.run) { console.error('usage: node topics.js --run <runDir> [--rater r1]'); process.exit(1); }
const runDir = path.resolve(args.run);
const man = readJson(path.join(runDir, 'manifest.json'));
const prereg = readJson(path.join(runDir, 'prereg.json'));
const pref = args.rater || 'r1';
const CRAFT_WORDS = /\b(animated|animation|2d|gripping|engaging|well|detailed|thorough|compelling|fascinating|boring|dramatic|shocking|deep[- ]dive|video essay|narrat\w*)\b/i;

const cards = {};
for (const sid of man.samples) {
  const files = fs.readdirSync(path.join(runDir, 'scores')).filter((f) => new RegExp(`^${sid}\\.r\\d+\\.json$`).test(f)).sort();
  const pick = files.find((f) => f.includes(`.${pref}.`)) || files[0];
  if (!pick) { console.error(`No rater output for ${sid}; finish wave 1 first.`); process.exit(1); }
  const card = (readJson(path.join(runDir, 'scores', pick)).topic_card || '').trim();
  if (!card) { console.error(`${pick} has no topic_card`); process.exit(1); }
  if (CRAFT_WORDS.test(card)) console.warn(`WARNING ${sid}: topic card mentions execution/format ("${card.match(CRAFT_WORDS)[0]}"); edit blind/topic_cards.json before launching if needed.`);
  cards[sid] = card;
}
const rand = rng((prereg.seed || 1) + 7);
const tOrder = shuffle(man.samples, rand);
const tidOf = Object.fromEntries(tOrder.map((s, i) => [s, `T${i + 1}`]));
writeJson(path.join(runDir, 'blind', 'topic_cards.json'), tOrder.map((s) => ({ tid: tidOf[s], card: cards[s] })));
writeJson(path.join(runDir, 'sealed', 'topic_key.json'), Object.fromEntries(tOrder.map((s) => [tidOf[s], s])));

const tids = tOrder.map((s) => tidOf[s]);
const comps = [];
tids.forEach((x, i) => tids.slice(i + 1).forEach((y) => { comps.push([x, y]); comps.push([y, x]); }));
const list = shuffle(comps, rand);
const out = path.join(runDir, 'scores', 'topic.json');
const task = path.join(runDir, 'tasks', 'topic.md');
fs.writeFileSync(task, [
  '# Blind topic comparison task', '',
  `1. Read the instructions: ${path.join(SKILL_ROOT, 'references', 'topic-prompt.md')}`,
  `2. Read the topic cards: ${path.join(runDir, 'blind', 'topic_cards.json')}`,
  `3. Audience: ${prereg.audience || man.audience}`,
  `4. Judge these ${list.length} comparisons, in this order:`,
  ...list.map(([x, y], k) => `   ${k + 1}. ${x} vs ${y}`),
  `5. Write your JSON to: ${out}`,
  `6. Validate: node "${path.join(__dirname, 'check.js')}" --file "${out}"  (fix and rewrite until it prints OK)`,
  '',
  'Read ONLY the files listed above. Do not list directories, glob, grep or open any other file: other files reveal the answers and would invalidate the experiment.', '',
].join('\n'), 'utf8');
man.topic_task = task;
man.topic_comparisons = list;
writeJson(path.join(runDir, 'manifest.json'), man);
console.log(`Topic cards written (${tids.length}); ${list.length} comparisons.\nTopic task: ${task}`);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.