Docs/Skills and code/skill within-channel-study

.claude/skills/within-channel-study/scripts/dossier.js

Code137 lines10.9 KBSkill: within-channel-study

#!/usr/bin/env node
// Builds the fact-only study dossier for one channel from the DB + the six sampled transcripts.
// No ratings, no judgments: just the numbers, title facts, nearest catalogue neighbours and compact transcript views
// that the analyst needs to read the six videos against the channel baseline.
//   node dossier.js --channel <name|handle|url> [--db data/x.db] [--transcripts-dir PATH]
// Writes transcripts/<channel>/_study/dossier.md
const fs = require('fs');
const path = require('path');
const { ROOT, parseArgs, channelFolderName, findChannel, loadVideos, median, quantile, rankVideos, parseTranscriptFile, fmtTs } = require('./lib');

const args = parseArgs(process.argv.slice(2));
if (!args.channel || args.channel === true) { console.error('usage: node dossier.js --channel <name|handle|url> [--db data/x.db]'); process.exit(1); }
const hits = findChannel(String(args.channel), args.db);
if (!hits.length) { console.error('Channel not found; run check-prereqs.js first.'); process.exit(4); }
const { channel, db, dbPath } = hits.map(h => ({ ...h, n: loadVideos(h.db, h.channel.channel_url).length })).sort((a, b) => b.n - a.n)[0];
const videos = loadVideos(db, channel.channel_url);
const { ranked, fallback } = rankVideos(videos);
const dir = path.join(path.resolve(args['transcripts-dir'] || path.join(ROOT, 'transcripts')), channelFolderName(channel.name));
const idxPath = path.join(dir, '_index.json');
if (!fs.existsSync(idxPath)) { console.error('No _index.json in ' + dir + '; run check-prereqs.js first.'); process.exit(3); }
const idx = JSON.parse(fs.readFileSync(idxPath, 'utf8'));
const byId = new Map(ranked.map(v => [v.video_id, v]));

const pct = (x, d = 0) => x == null ? 'n/a' : Number(x).toLocaleString('en-US', { maximumFractionDigits: d });
const mins = s => s == null ? 'n/a' : `${Math.floor(s / 60)}m${String(Math.round(s % 60)).padStart(2, '0')}s`;
const snip = (t, n) => { const s = String(t || '').replace(/\s+/g, ' ').trim(); return s.length > n ? s.slice(0, n) + '…' : s; };
const cell = t => String(t ?? '').replace(/\|/g, '\\|');
const label = f => (String(f.file).match(/^(best|median|worst)-\d+/) || [f.bucket])[0];

// ---- baseline ----
const vpds = ranked.map(v => v.vpd);
const durs = ranked.map(v => Number(v.duration_seconds)).filter(Number.isFinite);
const medVpd = median(vpds), p10 = quantile(vpds, 0.1), p90 = quantile(vpds, 0.9);
const medDur = median(durs);
const partRe = /(?:\bpart\s*\d+|\bep(?:isode)?\.?\s*\d+|\bchapter\s*\d+|भाग\s*\d+|पार्ट\s*\d+|#\d+|\b\d+\s*\/\s*\d+)/i;
const seriesShare = ranked.filter(v => partRe.test(v.title || '')).length / ranked.length;
const dates = ranked.map(v => Date.parse(v.published_at_normalized)).filter(Number.isFinite);
const spanDays = (Math.max(...dates) - Math.min(...dates)) / 86400000;

// ---- helpers ----
const tokens = t => new Set((String(t || '').toLowerCase().match(/[\p{L}\p{N}]{3,}/gu) || []));
const jaccard = (a, b) => { let i = 0; for (const x of a) if (b.has(x)) i++; return i / (a.size + b.size - i || 1); };
function titleFacts(t) {
  t = String(t || '');
  const dev = (t.match(/[ऀ-ॿ]/g) || []).length, lat = (t.match(/[A-Za-z]/g) || []).length;
  return {
    chars: [...t].length, words: t.split(/\s+/).filter(Boolean).length,
    script: dev && lat ? 'mixed Devanagari+Latin' : dev ? 'Devanagari' : 'Latin',
    digits: /\d/.test(t), currency: /[₹$€£]|\b(?:crore|lakh|billion|million)\b/i.test(t), question: /[?？]/.test(t),
    segments: t.split(/\s*[|—–:]\s+/).filter(Boolean).length, allcaps: (t.match(/\b[A-Z]{3,}\b/g) || []).length,
    emoji: /[\u{1F300}-\u{1FAFF}☀-➿❌✅]/u.test(t), seriesMarker: partRe.test(t)
  };
}
function timeSlices(segs, n) {
  if (!segs.length) return [];
  const total = segs[segs.length - 1].t + 5, out = [];
  for (let k = 0; k < n; k++) {
    const lo = (total * k) / n, hi = (total * (k + 1)) / n;
    const s = segs.filter(x => x.t >= lo && x.t < hi);
    out.push({ lo, text: s.map(x => x.text).join(' ') });
  }
  return out;
}
const window = (segs, from, to) => segs.filter(s => s.t >= from && s.t < to).map(s => `[${s.ts}] ${s.text}`).join('\n');

// ---- build ----
const samples = (idx.files || []).map(f => {
  const v = byId.get(f.video_id) || { ...f, vpd: null, age_days: null, rank: f.rank, rank_total: idx.ranking?.eligible };
  const tr = parseTranscriptFile(path.join(dir, f.file));
  return { f, v, tr };
}).sort((a, b) => ['best', 'median', 'worst'].indexOf(a.f.bucket) - ['best', 'median', 'worst'].indexOf(b.f.bucket) || a.f.file.localeCompare(b.f.file));

const L = [];
L.push(`# Study dossier: ${channel.name} (${channel.handle || ''})`, '');
L.push(`Generated ${new Date().toISOString()} from \`${path.relative(ROOT, dbPath)}\` and \`${path.relative(ROOT, dir)}\`. Facts only: no ratings or interpretation in this file.`, '');
L.push('## 1. Channel baseline (what is held constant)', '');
L.push(`- Subscribers ${pct(channel.subscriber_count)}; ${videos.length} saved videos; ${ranked.length} rankable${fallback ? ' (**fallback pool: includes videos under 14 days old**)' : ''}; catalogue spans ${pct(spanDays)} days.`);
L.push(`- Views/day (age floored at 30 days): median ${pct(medVpd, 1)}, p10 ${pct(p10, 1)}, p90 ${pct(p90, 1)}; p90/p10 spread **${p10 ? pct(p90 / p10, 1) : 'n/a'}x**. A large spread means video-level factors matter on this channel; a small one means the channel baseline dominates.`);
L.push(`- Typical duration (median) ${mins(medDur)}; ${pct(seriesShare * 100)}% of titles carry a part/episode marker.`, '');
L.push('## 2. The six videos', '');
L.push('| Bucket | Title | Views/day | x channel median | Rank | Age (d) | Duration | Dur vs median | Date precision |', '|---|---|---|---|---|---|---|---|---|');
for (const { f, v } of samples) {
  const d = Number(v.duration_seconds ?? f.duration_seconds);
  L.push(`| ${label(f)} | ${cell(snip(v.title ?? f.title, 90))} | ${pct(v.vpd, 1)} | ${v.vpd && medVpd ? pct(v.vpd / medVpd, 2) + 'x' : 'n/a'} | ${v.rank}/${v.rank_total} | ${pct(v.age_days)} | ${mins(d)} | ${medDur && d ? pct(d / medDur, 2) + 'x' : 'n/a'} | ${v.published_at_precision ?? 'n/a'} |`);
}
L.push('', '### Automatic confound flags', '');
for (const { f, v } of samples) {
  const d = Number(v.duration_seconds ?? f.duration_seconds), flags = [];
  if (v.age_days != null && v.age_days < 45) flags.push(`young (${pct(v.age_days)} days): views/day is still moving and the 30-day floor flatters or punishes it`);
  if (medDur && d && (d / medDur > 1.8 || d / medDur < 0.55)) flags.push(`unusual length (${pct(d / medDur, 2)}x channel median)`);
  if (partRe.test(v.title ?? f.title ?? '')) flags.push('series/part marker in title (position in a series is a separate driver)');
  if (v.published_at_precision && !String(v.published_at_precision).startsWith('exact')) flags.push(`publish date is only ${v.published_at_precision}: age and views/day are approximate`);
  L.push(`- **${label(f)} ${v.video_id ?? f.video_id}**: ${flags.length ? flags.join('; ') : 'no automatic flag'}`);
}
L.push('', '## 3. Packaging facts (titles)', '', '| Bucket | Chars | Words | Script | Digits | Currency/magnitude | ? | Segments | ALLCAPS | Emoji | Series |', '|---|---|---|---|---|---|---|---|---|---|---|');
for (const { f, v } of samples) {
  const t = titleFacts(v.title ?? f.title);
  L.push(`| ${label(f)} | ${t.chars} | ${t.words} | ${t.script} | ${t.digits ? 'y' : '-'} | ${t.currency ? 'y' : '-'} | ${t.question ? 'y' : '-'} | ${t.segments} | ${t.allcaps} | ${t.emoji ? 'y' : '-'} | ${t.seriesMarker ? 'y' : '-'} |`);
}
L.push('', '### Nearest catalogue neighbours (same topic words elsewhere on the channel)', '', 'Token overlap only. Use it to see whether the *topic* performs consistently on this channel, independent of this one video.', '');
for (const { f, v } of samples) {
  const me = tokens(v.title ?? f.title);
  const near = ranked.filter(x => x.video_id !== (v.video_id ?? f.video_id)).map(x => ({ x, s: jaccard(me, tokens(x.title)) })).filter(o => o.s > 0.12).sort((a, b) => b.s - a.s).slice(0, 5);
  L.push(`- **${label(f)}: ${snip(v.title ?? f.title, 70)}** (${pct(v.vpd, 1)} views/day)`);
  if (!near.length) L.push('  - no close neighbours: this topic is unique on the channel');
  for (const { x, s } of near) L.push(`  - ${pct(x.vpd, 1)} views/day (rank ${x.rank}/${x.rank_total}, overlap ${pct(s * 100)}%): ${snip(x.title, 80)}`);
}
L.push('', '## 4. Transcript views', '', 'Full text lives in the sample files. Each video: opening 90 s, a 10-slice skeleton, and the last 60 s. Read more of the middle only when a hypothesis needs it.', '');
for (const { f, v, tr } of samples) {
  const segs = tr.segs, end = segs.length ? segs[segs.length - 1].t : 0;
  const words = segs.map(s => s.text).join(' ').split(/\s+/).filter(Boolean).length;
  L.push(`### ${label(f)} · ${v.title ?? f.title}`, '');
  L.push(`File: \`${path.relative(ROOT, tr.file).replace(/\\/g, '/')}\` · ${segs.length} caption segments · ~${pct(words)} words · ${fmtTs(end)} long · ${pct(end ? words / (end / 60) : 0)} words/min · language: ${snip(tr.field('Language'), 80)}`, '');
  L.push('**Opening (first 90 s):**', '```', window(segs, 0, 90) || '(none)', '```', '');
  L.push('**Skeleton (start of each tenth of the runtime):**', '');
  timeSlices(segs, 10).forEach((s, i) => L.push(`${i + 1}. [${fmtTs(s.lo)}] ${snip(s.text, 220)}`));
  L.push('', '**Ending (last 60 s):**', '```', window(segs, Math.max(0, end - 60), end + 1) || '(none)', '```', '');
}
const outDir = path.join(dir, '_study');
fs.mkdirSync(outDir, { recursive: true });
fs.writeFileSync(path.join(outDir, 'dossier.md'), L.join('\n'));
console.log('Wrote ' + path.relative(ROOT, path.join(outDir, 'dossier.md')));

// ---- whole-catalogue title ledger: lets the analyst check an observation against every video by reading ----
const sampledLabel = new Map(samples.map(({ f, v }) => [v.video_id ?? f.video_id, label(f)]));
const n = ranked.length;
const T = ['# Catalogue title ledger: ' + channel.name, '',
  `All ${n} rankable videos, best to worst by views/day (age floored at 30 days). Tier: **TOP** = top 20%, **MID** = middle 60%, **LOW** = bottom 20%. Videos in the six-video sample are tagged.`,
  'Use this to find supporting and counter-examples for an observation by reading titles and counting, e.g. "of the TOP 10 titles, how many name a loss or disaster vs the LOW 10?".', '',
  '| # | Tier | Views/day | x median | Age (d) | Min | Title | Sample |', '|---|---|---|---|---|---|---|---|'];
ranked.forEach((v, i) => {
  const tier = i < n * 0.2 ? 'TOP' : i >= n * 0.8 ? 'LOW' : 'MID';
  T.push(`| ${i + 1} | ${tier} | ${pct(v.vpd, 1)} | ${medVpd ? pct(v.vpd / medVpd, 2) : 'n/a'}x | ${pct(v.age_days)} | ${v.duration_seconds ? Math.round(v.duration_seconds / 60) : 'n/a'} | ${cell(snip(v.title, 110))} | ${sampledLabel.get(v.video_id) ?? ''} |`);
});
fs.writeFileSync(path.join(outDir, 'catalogue-titles.md'), T.join('\n'));
console.log('Wrote ' + path.relative(ROOT, path.join(outDir, 'catalogue-titles.md')));

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.