Docs/Skills and code/skill script-cohort-analysis
.claude/skills/script-cohort-analysis/scripts/perturb.js
What the file says about itself
Calibration ("damage test"): damage ONE property of a transcript and check that blind raters move the measures for that property, and leave the others alone. Results feed the rule scorecard (evidence line 3).
#!/usr/bin/env node
// Calibration ("damage test"): damage ONE property of a transcript and check that blind raters
// move the measures for that property, and leave the others alone. Results feed the rule
// scorecard (evidence line 3).
//
// node perturb.js make --run <ratedRunDir> [--samples auto|S5,S2] [--out <run>/calibration] [--raters 3] [--seed N]
// node perturb.js analyze --dir <run>/calibration
//
// --samples auto picks the two samples with the highest blind checklist composite (no
// performance data is read), so the damage starts from scripts that pass many checks.
const fs = require('fs');
const path = require('path');
const { readJson, writeJson, parseArgs, rng, shuffle, mean } = require('./lib');
const { raterFeatures } = require('./raterFeatures');
const { renderMarkdown } = require('./checklist');
const SKILL_ROOT = path.resolve(__dirname, '..');
const BLIND_RULE = 'Read ONLY the files listed above. Do not list directories, glob, grep or open any other file: other files reveal the answers and would invalidate the experiment.';
// Generic, topic-free channel preamble (Hindi). Two ~30s chunks.
const PREAMBLE = [
'नमस्ते दोस्तों, स्वागत है आपका हमारे चैनल पर। आज की वीडियो में हम एक बहुत ही जरूरी टॉपिक के बारे में बात करने वाले हैं। तो वीडियो को अंत तक जरूर देखिए, और अगर आप चैनल पर नए हैं तो चैनल को सब्सक्राइब कर लीजिए और बेल आइकन दबा दीजिए ताकि आपको हमारी हर नई वीडियो की नोटिफिकेशन मिलती रहे।',
'इस वीडियो को बनाने में हमारी पूरी टीम ने बहुत मेहनत की है, इसलिए वीडियो को लाइक जरूर करें और कमेंट में बताएं कि आपको यह टॉपिक कैसा लगा। और हां, अपने दोस्तों के साथ भी इस वीडियो को शेयर कीजिए। तो चलिए बिना किसी देरी के, शुरू करते हैं आज की वीडियो।',
];
const RECALL = 'जैसा कि हमने पहले भी बताया,';
const GUARD_MAX = 0.2;
// Pre-registered expectations. A variant is detected if ANY target moves as stated and ALL guards move < GUARD_MAX.
const VARIANTS = {
hook_preamble: { desc: '60s generic greeting/subscribe preamble prepended',
targets: [{ feature: 'chk_hook', dir: -1, min: 0.2 }], guards: ['chk_causality', 'chk_credibility', 'chk_clarity'] },
shuffled: { desc: 'middle chunks permuted in 1-minute blocks',
targets: [{ feature: 'chk_causality', dir: -1, min: 0.2 }, { feature: 'ext_forward_refs', dir: 1, min: 1 }, { feature: 'ext_dependency_density', dir: -1, min: 0.15 }], guards: ['chk_hook', 'chk_credibility', 'chk_voice'] },
padded: { desc: 'a recalled earlier chunk inserted after every 3rd chunk',
targets: [{ feature: 'chk_value_density', dir: -1, min: 0.2 }, { feature: 'ext_filler_ratio', dir: 1, min: 0.05 }], guards: ['chk_hook', 'chk_credibility', 'chk_voice'] },
truncated: { desc: 'last 30% of chunks removed',
targets: [{ feature: 'chk_payoff', dir: -1, min: 0.2 }], guards: ['chk_hook', 'chk_voice', 'chk_clarity'] },
};
function readBlind(file) {
const lines = fs.readFileSync(file, 'utf8').split(/\r?\n/);
const chunks = lines.map((l) => l.match(/^\[(\d{1,2}(?::\d{2}){1,2})\]\s*(.*)$/)).filter(Boolean).map((m) => m[2]);
const d = lines[1].match(/Duration: (\d+)m (\d+)s/);
return { header: lines[1], chunks, duration: d ? Number(d[1]) * 60 + Number(d[2]) : chunks.length * 30 };
}
const ts = (i) => { const s = i * 30; return `${String(Math.floor(s / 60)).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`; };
const raterFilesFor = (dir, sid) => fs.readdirSync(path.join(dir, 'scores')).filter((f) => new RegExp(`^${sid}\\.r\\d+\\.json$`).test(f)).map((f) => readJson(path.join(dir, 'scores', f)));
function apply(kind, chunks, rand) {
const head = 2, tail = 3;
if (kind === 'hook_preamble') return [...PREAMBLE, ...chunks];
if (kind === 'truncated') return chunks.slice(0, Math.round(chunks.length * 0.7));
if (kind === 'padded') {
const out = [];
chunks.forEach((c, i) => {
out.push(c);
if (i >= head && i < chunks.length - tail && (i - head) % 3 === 2) out.push(`${RECALL} ${chunks[Math.max(1, i - 2 - Math.floor(rand() * 3))]}`);
});
return out;
}
if (kind === 'shuffled') {
const mid = chunks.slice(head, chunks.length - tail);
const blocks = [];
for (let i = 0; i < mid.length; i += 2) blocks.push(mid.slice(i, i + 2));
let perm;
do { perm = shuffle(blocks.map((_, i) => i), rand); } while (perm.some((p, i) => p === i) && blocks.length > 3);
return [...chunks.slice(0, head), ...perm.flatMap((p) => blocks[p]), ...chunks.slice(chunks.length - tail)];
}
throw new Error(kind);
}
function make(args) {
const run = path.resolve(args.run);
const out = path.resolve(args.out || path.join(run, 'calibration'));
const nR = Number(args.raters || 3);
const rand = rng(Number(args.seed || 99));
let bases;
if (!args.samples || args.samples === 'auto') {
const man = readJson(path.join(run, 'manifest.json'));
const comp = man.samples.map((s) => {
const d = readBlind(path.join(run, 'blind', `${s}.txt`)).duration;
return [s, mean(raterFilesFor(run, s).map((r) => raterFeatures(r, d).chk_composite))];
}).sort((a, b) => b[1] - a[1]);
bases = comp.slice(0, 2).map(([s]) => s);
console.log(`auto-selected bases by blind checklist composite: ${comp.map(([s, v]) => `${s} ${v.toFixed(2)}`).join(', ')} -> ${bases.join(', ')}`);
} else bases = String(args.samples).split(',');
const items = shuffle(bases.flatMap((b) => Object.keys(VARIANTS).map((kind) => ({ base: b, kind }))), rand);
for (const d of ['blind', 'scores', 'tasks', 'sealed']) fs.mkdirSync(path.join(out, d), { recursive: true });
fs.writeFileSync(path.join(out, 'checklist.md'), renderMarkdown(), 'utf8');
const sealed = {}, tasks = [];
items.forEach((it, i) => {
const sid = `S${i + 1}`;
const src = readBlind(path.join(run, 'blind', `${it.base}.txt`));
const chunks = apply(it.kind, src.chunks, rand);
const dur = chunks.length * 30;
const lang = (src.header.match(/Transcript language: ([^|]+)/) || [, 'unknown'])[1].trim();
fs.writeFileSync(path.join(out, 'blind', `${sid}.txt`), [
`BLIND SAMPLE ${sid}`,
`Duration: ${Math.floor(dur / 60)}m ${String(dur % 60).padStart(2, '0')}s | Transcript language: ${lang} | Chunks are ~30s, timestamps mark chunk starts`,
'', ...chunks.map((c, k) => `[${ts(k)}] ${c}`), '',
].join('\n'), 'utf8');
sealed[sid] = { base: it.base, kind: it.kind, ...VARIANTS[it.kind] };
for (let r = 1; r <= nR; r++) {
const o = path.join(out, 'scores', `${sid}.r${r}.json`);
const p = path.join(out, 'tasks', `rate_${sid}_r${r}.md`);
fs.writeFileSync(p, [
`# Blind rating task: ${sid}, rater ${r}`, '',
`1. Read the rater instructions: ${path.join(SKILL_ROOT, 'references', 'rater-prompt.md')}`,
`2. Read the definitions: ${path.join(SKILL_ROOT, 'references', 'rubric.md')}`,
`3. Read the checklist: ${path.join(out, 'checklist.md')}`,
`4. Read the transcript: ${path.join(out, 'blind', `${sid}.txt`)}`,
`5. Write your JSON to: ${o}`,
`6. Validate: node "${path.join(__dirname, 'check.js')}" --file "${o}" (fix and rewrite until it prints OK)`,
'', 'No extra hypothesis questions this run (`hypotheses` = {}).', '', BLIND_RULE, '',
].join('\n'), 'utf8');
tasks.push(p);
}
});
const baseRubric = fs.existsSync(path.join(run, 'prereg.json')) ? readJson(path.join(run, 'prereg.json')).rubric_hash : null;
writeJson(path.join(out, 'sealed', 'variants.json'), sealed);
writeJson(path.join(out, 'manifest.json'), { baseline_run: run, rubric_hash: baseRubric, samples: Object.keys(sealed), raters: nR, rater_tasks: tasks, variants: VARIANTS, guard_max: GUARD_MAX });
console.log(`Calibration set: ${out}\n${tasks.length} rater tasks:`);
tasks.forEach((t) => console.log(' ' + t));
}
function analyze(args) {
const dir = path.resolve(args.dir);
const man = readJson(path.join(dir, 'manifest.json'));
const sealed = readJson(path.join(dir, 'sealed', 'variants.json'));
const feats = (d, sid) => {
const dur = readBlind(path.join(d, 'blind', `${sid}.txt`)).duration;
return raterFilesFor(d, sid).map((r) => raterFeatures(r, dur));
};
const avg = (fs_, k) => mean(fs_.map((f) => f[k]));
const sgn = (v) => (v > 0 ? '+' : '') + v.toFixed(2);
const variants = [];
for (const sid of Object.keys(sealed).sort((a, b) => sealed[a].kind.localeCompare(sealed[b].kind) || sealed[a].base.localeCompare(sealed[b].base))) {
const v = sealed[sid];
const vf = feats(dir, sid), bf = feats(man.baseline_run, v.base);
if (!vf.length) { variants.push({ sid, ...v, missing: true }); continue; }
const keys = Object.keys(vf[0]).filter((k) => k.startsWith('chk_') || k.startsWith('ext_'));
const deltas = Object.fromEntries(keys.map((k) => [k, avg(vf, k) - avg(bf, k)]));
const target_results = Object.fromEntries(v.targets.map((t) => [t.feature, t.dir * deltas[t.feature] >= t.min]));
const guard_results = Object.fromEntries(v.guards.map((g) => [g, Math.abs(deltas[g]) < GUARD_MAX]));
const pass = Object.values(target_results).some(Boolean) && Object.values(guard_results).every(Boolean);
variants.push({ sid, base: v.base, kind: v.kind, desc: v.desc, targets: v.targets, guards: v.guards, deltas, target_results, guard_results, pass, raters: vf.length });
}
const done = variants.filter((v) => !v.missing);
const cal = { rubric_hash: man.rubric_hash, baseline_run: man.baseline_run, guard_max: GUARD_MAX, passed: done.filter((v) => v.pass).length, total: done.length, variants: done };
writeJson(path.join(dir, 'calibration.json'), cal);
const L = ['# Calibration (damage test)', '', `Baseline: ${man.baseline_run}`, `Rule: a variant is detected if ANY target moves by at least its minimum in the stated direction AND every guard moves less than ${GUARD_MAX}.`, '',
'| Variant | Base | Targets (Δ, ok?) | Guards (Δ, ok?) | Result |', '|---|---|---|---|---|'];
for (const v of done) {
L.push(`| ${v.kind} | ${v.base} | ${v.targets.map((t) => `${t.feature} ${sgn(v.deltas[t.feature])} ${v.target_results[t.feature] ? '✓' : '✗'}`).join(', ')} | ${v.guards.map((g) => `${g} ${sgn(v.deltas[g])} ${v.guard_results[g] ? '✓' : '✗'}`).join(', ')} | ${v.pass ? 'PASS' : Object.values(v.target_results).some(Boolean) ? 'off-target drift' : 'MISSED'} |`);
}
const byKind = {};
for (const v of done) (byKind[v.kind] = byKind[v.kind] || []).push(v.pass);
L.push('', `**${cal.passed}/${cal.total} manipulations detected cleanly.** By type: ${Object.entries(byKind).map(([k, x]) => `${k} ${x.filter(Boolean).length}/${x.length}`).join(', ')}.`);
L.push('', '## All checklist-dimension deltas', '', '| Variant | Base | ' + Object.keys(done[0].deltas).filter((k) => k.startsWith('chk_')).map((k) => k.slice(4)).join(' | ') + ' |', '|---|---|' + Object.keys(done[0].deltas).filter((k) => k.startsWith('chk_')).map(() => '---').join('|') + '|');
for (const v of done) L.push(`| ${v.kind} | ${v.base} | ${Object.keys(v.deltas).filter((k) => k.startsWith('chk_')).map((k) => sgn(v.deltas[k])).join(' | ')} |`);
fs.writeFileSync(path.join(dir, 'calibration.md'), L.join('\n') + '\n', 'utf8');
console.log(L.join('\n'));
}
const [cmd] = process.argv.slice(2);
const args = parseArgs(process.argv.slice(3));
if (cmd === 'make' && args.run) make(args);
else if (cmd === 'analyze' && args.dir) analyze(args);
else { console.error('usage: node perturb.js make --run <ratedRun> [--samples auto|S5,S2] [--out <dir>] [--raters 3] | analyze --dir <dir>'); process.exit(1); }
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.