Docs/Skills and code/skill within-channel-study

.claude/skills/within-channel-study/scripts/probe.js

Code88 lines6.0 KBSkill: within-channel-study

#!/usr/bin/env node
// Tests title/metadata hypotheses on the WHOLE mined catalogue (up to 300 videos) instead of the six sampled ones.
//   node probe.js --channel <name|handle|url> [--db data/x.db]
//        [--regex "name=pattern"]...   custom title feature (case-insensitive, unicode); repeatable
//        [--token word1,word2]         shorthand: one feature per token (title contains it)
//        [--perms 5000] [--seed 1]
// Output: for each feature, median views/day with vs without it, the ratio, and a two-sided permutation p.
// Writes transcripts/<channel>/_study/probe.md as well as printing it.
const fs = require('fs');
const path = require('path');
const { ROOT, parseArgs, channelFolderName, findChannel, loadVideos, median, rankVideos } = require('./lib');

const args = parseArgs(process.argv.slice(2));
if (!args.channel || args.channel === true) { console.error('usage: node probe.js --channel <name> [--regex "name=pattern"] [--token a,b]'); process.exit(1); }
const hits = findChannel(String(args.channel), args.db);
if (!hits.length) { console.error('Channel not found; run check-prereqs.js first.'); process.exit(4); }
const { channel, db } = hits.map(h => ({ ...h, n: loadVideos(h.db, h.channel.channel_url).length })).sort((a, b) => b.n - a.n)[0];
const { ranked, fallback } = rankVideos(loadVideos(db, channel.channel_url));
if (ranked.length < 10) { console.error(`Only ${ranked.length} rankable videos; a catalogue probe is not meaningful.`); process.exit(5); }

let seed = Number(args.seed || 1) >>> 0;
const rnd = () => { seed = (seed + 0x6d2b79f5) >>> 0; let t = seed; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };
const PERMS = Number(args.perms || 5000);
const medDur = median(ranked.map(v => Number(v.duration_seconds)));
const medLen = median(ranked.map(v => [...String(v.title || '')].length));
const partRe = /(?:\bpart\s*\d+|\bep(?:isode)?\.?\s*\d+|\bchapter\s*\d+|भाग\s*\d+|पार्ट\s*\d+|#\d+|\b\d+\s*\/\s*\d+)/i;

const features = [
  ['title has a digit', v => /\d/.test(v.title || '')],
  ['title has currency / magnitude word', v => /[₹$€£]|\b(?:crore|lakh|billion|million)\b/i.test(v.title || '')],
  ['title is a question (?)', v => /[?？]/.test(v.title || '')],
  ['title has a pipe/colon/dash segment', v => /\s[|—–:]\s/.test(v.title || '')],
  ['title longer than channel median', v => [...String(v.title || '')].length > medLen],
  ['title has series/part marker', v => partRe.test(v.title || '')],
  ['title contains Devanagari', v => /[ऀ-ॿ]/.test(v.title || '')],
  ['title has ALLCAPS word', v => /\b[A-Z]{3,}\b/.test(v.title || '')],
  ['video longer than channel median', v => Number(v.duration_seconds) > medDur]
];
for (const spec of [].concat(args.regex || [])) {
  const eq = String(spec).indexOf('=');
  if (eq < 1) continue;
  const re = new RegExp(spec.slice(eq + 1), 'iu');
  features.push([`custom: ${spec.slice(0, eq)}`, v => re.test(v.title || '')]);
}
for (const tok of String(args.token || '').split(',').map(s => s.trim()).filter(Boolean)) {
  const t = tok.toLowerCase();
  features.push([`title contains "${tok}"`, v => String(v.title || '').toLowerCase().includes(t)]);
}

function permP(x, y) {
  const obs = Math.abs(median(x) - median(y)), all = x.concat(y), nx = x.length;
  let ge = 0;
  for (let p = 0; p < PERMS; p++) {
    for (let i = all.length - 1; i > 0; i--) { const j = Math.floor(rnd() * (i + 1)); [all[i], all[j]] = [all[j], all[i]]; }
    if (Math.abs(median(all.slice(0, nx)) - median(all.slice(nx))) >= obs) ge++;
  }
  return (ge + 1) / (PERMS + 1);
}
const f1 = x => x == null ? 'n/a' : Number(x).toLocaleString('en-US', { maximumFractionDigits: 1 });
const f2 = x => x == null ? 'n/a' : Number(x).toLocaleString('en-US', { maximumFractionDigits: 2 });

const rows = [];
for (const [name, fn] of features) {
  const yes = ranked.filter(fn).map(v => v.vpd), no = ranked.filter(v => !fn(v)).map(v => v.vpd);
  if (yes.length < 3 || no.length < 3) { rows.push({ name, n1: yes.length, n0: no.length, skip: true }); continue; }
  const m1 = median(yes), m0 = median(no);
  rows.push({ name, n1: yes.length, n0: no.length, m1, m0, ratio: m1 / m0, p: permP(yes.slice(), no.slice()) });
}

// Is the channel trending? (older vs newer half)
const byDate = [...ranked].sort((a, b) => Date.parse(a.published_at_normalized) - Date.parse(b.published_at_normalized));
const half = Math.floor(byDate.length / 2);
const olderMed = median(byDate.slice(0, half).map(v => v.vpd)), newerMed = median(byDate.slice(half).map(v => v.vpd));

const L = [];
L.push(`# Catalogue probe: ${channel.name}`, '');
L.push(`${ranked.length} rankable videos${fallback ? ' (fallback pool)' : ''}; metric = views/day (age floored at 30 d); ${PERMS} permutations, median difference, two-sided.`, '');
L.push(`Recency check: older half median ${f1(olderMed)} views/day, newer half ${f1(newerMed)} (${olderMed ? f1(newerMed / olderMed) : 'n/a'}x). If this is far from 1x, every feature below is confounded with *when* the video was published.`, '');
L.push('| Feature | n with | n without | median v/d with | without | ratio | p |', '|---|---|---|---|---|---|---|');
for (const r of rows) L.push(r.skip ? `| ${r.name} | ${r.n1} | ${r.n0} | too few in a group (<3) | | | |` : `| ${r.name} | ${r.n1} | ${r.n0} | ${f1(r.m1)} | ${f1(r.m0)} | ${f2(r.ratio)}x | ${r.p.toFixed(3)} |`);
const tested = rows.filter(r => !r.skip).length;
L.push('', `Read this as screening, not proof: ${tested} features tested, so about ${(tested * 0.05).toFixed(1)} will show p<0.05 by chance. Features are correlated with each other and with publish date. Treat ratio and sample size first, p second. Groups under ~15 videos are weak.`);
const out = L.join('\n');
console.log(out);
const outDir = path.join(ROOT, 'transcripts', channelFolderName(channel.name), '_study');
fs.mkdirSync(outDir, { recursive: true });
fs.writeFileSync(path.join(outDir, 'probe.md'), out);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.