Docs/Skills and code/src/
src/series-analysis/cli.js
import fs from "node:fs";
import path from "node:path";
import Database from "better-sqlite3";
import { Log } from "youtubei.js";
import { openStore, channelKeyOf } from "./store.js";
import { buildQueries, llmQueryPrompt } from "./queries.js";
import { scoreCandidates, scoreFit, selectLearnFrom, studyLists } from "./scoring.js";
import { ageDaysFrom, clamp01, failureHits, median, publishedFromText } from "./signals.js";
import { buildReport } from "./report.js";
Log.setLevel(Log.Level.NONE);
/**
* series-analysis: given a series idea (series/<id>.json), find videos with the same or adjacent packaging,
* fetch video + channel metadata, and rank which ones are worth learning from.
*
* node src/series-analysis/cli.js <command> --series=why-x-died [options]
*
* plan build the search queries [--max-queries=N] [--llm]
* search run pending queries on YouTube [--retry] [--results=N] [--limit=N]
* mine add matching videos from your existing research DB [--research-db=PATH] [--min-views=N]
* enrich fetch video metadata, channel metadata and channel baselines [--videos=N] [--channels=N] [--retry]
* judge optional LLM check of series fit on the shortlist [--top=N]
* rank score everything and pick the learn-from set [--top=N]
* transcripts fetch transcripts for the learn-from set [--top=N]
* cohorts pick the new vs established study cohorts and save verified transcripts [--size=N]
* report write the markdown + JSON report
* all plan, search, mine, enrich, rank, report (add --transcripts to include transcripts, --llm for LLM steps)
*/
const [command = "help", ...rest] = process.argv.slice(2);
const args = Object.fromEntries(rest.filter(a => a.startsWith("--")).map(a => {
const [k, ...v] = a.slice(2).split("=");
return [k, v.length ? v.join("=") : true];
}));
// The shared YouTube helpers narrate every request and every feed video; keep that behind --verbose.
if (!args.verbose) {
for (const level of ["log", "warn"]) {
const original = console[level].bind(console);
console[level] = (...a) => { if (!(typeof a[0] === "string" && /^\[(youtube|youtube-search|youtube-video|LLM)\b/.test(a[0]) && !/fail|unavailable/i.test(a[0]))) original(...a); };
}
}
const num = (name, fallback) => (args[name] != null && Number.isFinite(Number(args[name])) ? Number(args[name]) : fallback);
const sleep = ms => new Promise(r => setTimeout(r, ms));
const log = (msg, extra) => console.log(`[series] ${msg}`, ...(extra === undefined ? [] : [extra]));
const nowIso = () => new Date().toISOString();
if (command === "help" || !args.series) {
console.log(fs.readFileSync(new URL(import.meta.url), "utf8").match(/\/\*\*([\s\S]*?)\*\//)[1].replace(/^ \* ?/gm, ""));
process.exit(command === "help" ? 0 : 1);
}
const specPath = path.resolve("series", `${args.series}.json`);
if (!fs.existsSync(specPath)) throw new Error(`No series spec at ${specPath}`);
const spec = JSON.parse(fs.readFileSync(specPath, "utf8"));
const S = spec.id;
const db = openStore();
const budget = spec.budget ?? {};
// ---------------------------------------------------------------- plan
async function plan() {
const maxQueries = num("max-queries", budget.maxQueries ?? 70);
const queries = buildQueries(spec, maxQueries);
const insert = db.prepare("INSERT OR IGNORE INTO series_queries(series_id,query,tier,formula,anchor,origin) VALUES(?,?,?,?,?,?)");
let added = 0;
db.transaction(() => { for (const q of queries) added += insert.run(S, q.query, q.tier, q.formula ?? null, q.anchor ?? null, "template").changes; })();
if (args.llm) {
try {
const { callLLM, parseJsonObject } = await import("../llm.js");
const existing = db.prepare("SELECT query FROM series_queries WHERE series_id=?").all(S).map(r => r.query);
const p = llmQueryPrompt(spec, existing);
const { content, model } = await callLLM({ system: p.system, user: p.user, stream: false });
const items = parseJsonObject(content);
const valid = (Array.isArray(items) ? items : items?.queries ?? []).filter(i => typeof i?.query === "string" && i.query.trim());
let n = 0;
for (const i of valid) n += insert.run(S, i.query.trim(), ["exact", "adjacent-packaging", "adjacent-topic"].includes(i.tier) ? i.tier : "adjacent-packaging", null, null, `llm:${model}`).changes;
log(`LLM added ${n} queries (${valid.length} proposed)`);
} catch (e) { log(`LLM query expansion skipped: ${e.message}`); }
}
const byTier = db.prepare("SELECT tier, COUNT(*) n, SUM(status='pending') pending FROM series_queries WHERE series_id=? GROUP BY tier").all(S);
log(`plan: ${added} new queries`, byTier);
}
// ---------------------------------------------------------------- search
async function search() {
const { searchVideos } = await import("../youtube.js");
const statuses = args.retry ? "('pending','error')" : "('pending')";
let rows = db.prepare(`SELECT query, tier FROM series_queries WHERE series_id=? AND status IN ${statuses} ORDER BY rowid`).all(S);
if (num("limit", 0)) rows = rows.slice(0, num("limit", 0));
const perQuery = num("results", budget.resultsPerQuery ?? 25);
const upsert = db.prepare(`INSERT INTO series_videos(series_id,video_id,title,url,channel_name,channel_url,source,first_seen_at,search_views,search_published_text,search_duration_seconds,thumbnail_url)
VALUES(@series_id,@video_id,@title,@url,@channel_name,@channel_url,'search',@now,@view_count,@published_at_text,@duration_seconds,@thumbnail_url)
ON CONFLICT(series_id,video_id) DO UPDATE SET search_views=COALESCE(excluded.search_views,search_views), search_published_text=COALESCE(excluded.search_published_text,search_published_text),
search_duration_seconds=COALESCE(excluded.search_duration_seconds,search_duration_seconds), channel_url=COALESCE(channel_url,excluded.channel_url)`);
const hit = db.prepare("INSERT OR IGNORE INTO series_hits(series_id,video_id,query,position) VALUES(?,?,?,?)");
const done = db.prepare("UPDATE series_queries SET status='done', result_count=?, error=NULL, searched_at=? WHERE series_id=? AND query=?");
const fail = db.prepare("UPDATE series_queries SET status='error', error=?, searched_at=? WHERE series_id=? AND query=?");
let i = 0;
for (const { query, tier } of rows) {
i++;
try {
const videos = await searchVideos(query, perQuery);
db.transaction(() => videos.forEach((v, pos) => {
upsert.run({ ...v, series_id: S, now: nowIso() });
hit.run(S, v.video_id, query, pos + 1);
}))();
done.run(videos.length, nowIso(), S, query);
log(`search ${i}/${rows.length} [${tier}] "${query}" -> ${videos.length}`);
} catch (e) {
fail.run(String(e.message).slice(0, 500), nowIso(), S, query);
log(`search ${i}/${rows.length} FAILED "${query}": ${e.message}`);
}
await sleep(num("delay-ms", 400));
}
log("search done", db.prepare("SELECT COUNT(*) videos FROM series_videos WHERE series_id=?").get(S));
}
// ---------------------------------------------------------------- mine existing research
function mine() {
const dbFile = path.resolve(String(args["research-db"] ?? spec.researchDb ?? ""));
if (!fs.existsSync(dbFile)) return log(`mine: research DB not found (${dbFile}), skipping`);
const research = new Database(dbFile, { readonly: true });
const minViews = num("min-views", 20000);
// published_at_text is often relative to when that research was run; the normalized column is better when present.
const rows = research.prepare(`SELECT video_id,title,url,channel_name,channel_url,view_count,duration_seconds,COALESCE(published_at_normalized,published_at_text) AS published_at_text,thumbnail_url
FROM videos WHERE COALESCE(title,'')<>'' AND COALESCE(view_count,0)>=?`).all(minViews);
const insert = db.prepare(`INSERT OR IGNORE INTO series_videos(series_id,video_id,title,url,channel_name,channel_url,source,first_seen_at,search_views,search_published_text,search_duration_seconds,thumbnail_url)
VALUES(?,?,?,?,?,?, 'research-db',?,?,?,?,?)`);
let seen = 0, added = 0;
db.transaction(() => {
for (const v of rows) {
if (failureHits(v.title, spec.failureTerms).length === 0) continue;
if (scoreFit({ ...v, query_hits: 0 }, spec).fit < 0.45) continue;
seen++;
added += insert.run(S, v.video_id, v.title, v.url, v.channel_name, v.channel_url, nowIso(), v.view_count, v.published_at_text, v.duration_seconds, v.thumbnail_url).changes;
}
})();
research.close();
log(`mine: ${seen} on-series videos in ${path.basename(dbFile)}, ${added} new`);
}
// ---------------------------------------------------------------- enrich
function preliminary(v, hitCount) {
const fit = scoreFit({ title: v.title, query_hits: hitCount }, spec).fit;
const reach = clamp01(Math.log10((v.search_views ?? 0) + 1) / 7);
return 0.65 * fit + 0.35 * reach;
}
async function enrich() {
const { getVideoInfo, getChannel, getChannelVideos } = await import("../youtube-channel.js");
const topVideos = num("videos", budget.enrichTopVideos ?? 150);
const hitCounts = new Map(db.prepare("SELECT video_id, COUNT(*) n FROM series_hits WHERE series_id=? GROUP BY video_id").all(S).map(r => [r.video_id, r.n]));
const all = db.prepare("SELECT * FROM series_videos WHERE series_id=?").all(S);
const shortlist = all.map(v => ({ v, p: preliminary(v, hitCounts.get(v.video_id) ?? 0) }))
.sort((a, b) => b.p - a.p).slice(0, topVideos).map(x => x.v);
const todo = shortlist.filter(v => v.info_status == null || (args.retry && v.info_status === "error"));
log(`enrich: ${shortlist.length} shortlisted videos, ${todo.length} need metadata`);
const update = db.prepare(`UPDATE series_videos SET info_status='ok', info_error=NULL, info_fetched_at=@at, title=COALESCE(NULLIF(@title,''),title), channel_name=COALESCE(@channel_name,channel_name),
channel_id=@channel_id, views=@views, likes=@likes, published_at=@published_at, published_precision=@published_precision, duration_seconds=@duration_seconds,
category=@category, description=@description WHERE series_id=@series_id AND video_id=@video_id`);
const failed = db.prepare("UPDATE series_videos SET info_status='error', info_error=?, info_fetched_at=? WHERE series_id=? AND video_id=?");
let i = 0;
for (const v of todo) {
i++;
try {
// Full info is ~5s/video but is the only call that returns likes and the exact publish date; --basic trades both for speed.
let info;
try { info = await getVideoInfo(v.video_id, { fullInfo: !args.basic }); }
catch (e) { if (args.basic) throw e; info = await getVideoInfo(v.video_id); }
const fallback = publishedFromText(v.search_published_text, v.first_seen_at);
const rawPub = info.published_at ?? (/^\d{4}-\d{2}-\d{2}/.test(info.published_at_text ?? "") ? info.published_at_text : null);
const exact = rawPub && Number.isFinite(Date.parse(rawPub)) ? new Date(Date.parse(rawPub)).toISOString() : null;
update.run({
series_id: S, video_id: v.video_id, at: nowIso(), title: info.title, channel_name: info.channel_name, channel_id: info.video_channel_id,
views: info.view_count, likes: info.likes_count, published_at: exact ?? fallback.iso ?? null,
published_precision: exact ? "exact" : fallback.precision, duration_seconds: info.duration_seconds, category: info.category,
description: info.description?.slice(0, 4000) ?? null
});
} catch (e) {
failed.run(String(e.message).slice(0, 400), nowIso(), S, v.video_id);
log(` video ${v.video_id} failed: ${e.message}`);
}
if (i % 10 === 0) log(` videos ${i}/${todo.length}`);
await sleep(num("delay-ms", 300));
}
// Channels: the best videos' channels first, so the budget goes where the ranking will actually look.
const enriched = db.prepare("SELECT * FROM series_videos WHERE series_id=? AND info_status='ok'").all(S);
const order = new Map();
for (const v of enriched) {
const key = channelKeyOf(v);
if (!key) continue;
const p = preliminary(v, hitCounts.get(v.video_id) ?? 0);
const cur = order.get(key);
if (!cur || p > cur.p) order.set(key, { key, p, url: v.channel_url ?? `https://www.youtube.com/channel/${v.channel_id}`, id: v.channel_id });
}
const have = new Set(db.prepare("SELECT channel_key FROM series_channels WHERE series_id=? AND (status='ok' OR ?=0)").all(S, args.retry ? 1 : 0).map(r => r.channel_key));
const channels = [...order.values()].sort((a, b) => b.p - a.p).slice(0, num("channels", budget.enrichTopChannels ?? 60)).filter(c => !have.has(c.key));
log(`enrich: ${channels.length} channels to fetch`);
const upsertChannel = db.prepare(`INSERT INTO series_channels(series_id,channel_key,channel_url,channel_id,name,handle,description,country,subscriber_count,video_count,total_view_count,joined_date,is_verified,keywords_json,baseline_n,baseline_median_views,baseline_p75_views,baseline_note,status,error,fetched_at)
VALUES(@series_id,@channel_key,@channel_url,@channel_id,@name,@handle,@description,@country,@subscriber_count,@video_count,@total_view_count,@joined_date,@is_verified,@keywords_json,@baseline_n,@baseline_median_views,@baseline_p75_views,@baseline_note,@status,@error,@fetched_at)
ON CONFLICT(series_id,channel_key) DO UPDATE SET channel_url=excluded.channel_url,channel_id=excluded.channel_id,name=excluded.name,handle=excluded.handle,description=excluded.description,country=excluded.country,subscriber_count=excluded.subscriber_count,video_count=excluded.video_count,total_view_count=excluded.total_view_count,joined_date=excluded.joined_date,is_verified=excluded.is_verified,keywords_json=excluded.keywords_json,baseline_n=excluded.baseline_n,baseline_median_views=excluded.baseline_median_views,baseline_p75_views=excluded.baseline_p75_views,baseline_note=excluded.baseline_note,status=excluded.status,error=excluded.error,fetched_at=excluded.fetched_at`);
const baselineN = budget.channelBaselineVideos ?? 40;
let j = 0;
for (const c of channels) {
j++;
const base = { series_id: S, channel_key: c.key, channel_url: c.url, channel_id: c.id ?? null, fetched_at: nowIso() };
try {
const ch = await getChannel(c.url);
let baseline = { n: 0, median: null, p75: null, note: null };
try {
const feed = await getChannelVideos(ch, undefined, baselineN);
// Videos under 30 days old haven't finished earning views; including them drags the median down and inflates every "outlier".
const mature = feed.filter(f => {
const age = ageDaysFrom(publishedFromText(f.published_at_text, nowIso()).iso);
return age != null && age >= 30 && f.view_count != null;
}).map(f => f.view_count);
baseline = { n: mature.length, median: median(mature), p75: percentile(mature, 0.75), note: `latest ${feed.length} uploads, ${mature.length} older than 30 days` };
} catch (e) { baseline.note = `baseline failed: ${e.message}`.slice(0, 200); }
upsertChannel.run({
...base, channel_id: ch.channel_id, name: ch.name, handle: ch.handle, description: ch.description?.slice(0, 2000) ?? null, country: ch.country,
subscriber_count: ch.subscriber_count, video_count: ch.video_count, total_view_count: ch.total_view_count, joined_date: ch.joined_date,
is_verified: ch.is_verified == null ? null : Number(ch.is_verified), keywords_json: JSON.stringify(ch.keywords ?? []),
baseline_n: baseline.n, baseline_median_views: baseline.median, baseline_p75_views: baseline.p75, baseline_note: baseline.note, status: "ok", error: null
});
log(` channel ${j}/${channels.length}: ${ch.name} subs=${ch.subscriber_count_text} baseline=${baseline.median ?? "n/a"} (n=${baseline.n})`);
} catch (e) {
upsertChannel.run({ ...base, name: null, handle: null, description: null, country: null, subscriber_count: null, video_count: null, total_view_count: null, joined_date: null, is_verified: null, keywords_json: null, baseline_n: 0, baseline_median_views: null, baseline_p75_views: null, baseline_note: null, status: "error", error: String(e.message).slice(0, 400) });
log(` channel ${c.url} failed: ${e.message}`);
}
await sleep(num("delay-ms", 400));
}
}
function percentile(values, p) {
const v = values.filter(Number.isFinite).sort((a, b) => a - b);
if (!v.length) return null;
return v[Math.min(v.length - 1, Math.floor(p * v.length))];
}
// ---------------------------------------------------------------- optional LLM judge
async function judge() {
const { callLLM, parseJsonObject } = await import("../llm.js");
const rows = db.prepare(`SELECT v.video_id,v.title,v.channel_name,v.description FROM series_videos v
LEFT JOIN series_llm_fit f ON f.series_id=v.series_id AND f.video_id=v.video_id
WHERE v.series_id=? AND v.info_status='ok' AND f.video_id IS NULL`).all(S);
const hitCounts = new Map(db.prepare("SELECT video_id, COUNT(*) n FROM series_hits WHERE series_id=? GROUP BY video_id").all(S).map(r => [r.video_id, r.n]));
const top = rows.map(r => ({ r, f: scoreFit({ title: r.title, query_hits: hitCounts.get(r.video_id) ?? 1 }, spec).fit }))
.filter(x => x.f >= 0.3).sort((a, b) => b.f - a.f).slice(0, num("top", 60)).map(x => x.r);
const save = db.prepare("INSERT OR REPLACE INTO series_llm_fit(series_id,video_id,fit,subject,packaging,note,model,judged_at) VALUES(?,?,?,?,?,?,?,?)");
for (let i = 0; i < top.length; i += 10) {
const batch = top.slice(i, i + 10);
const user = `Series idea: "${spec.name}". ${spec.premise}
For each video decide how well it matches the series: a documentary-style explainer about why ONE specific, well-known company or brand collapsed / failed / died.
Score fit 0..1 (1 = exactly that; 0.5 = same disaster framing but about a person, scam, sector or event; 0 = unrelated, clip, podcast chatter, news).
Return JSON array: [{"id": string, "fit": number, "subject": string|null, "packaging": string (<=12 words: the hook/framing used in the title), "note": string (<=15 words)}]
Videos:
${batch.map(b => `- id=${b.video_id} | ${b.title} | channel: ${b.channel_name} | ${(b.description ?? "").replace(/\s+/g, " ").slice(0, 200)}`).join("\n")}`;
try {
const { content, model } = await callLLM({ system: "You judge YouTube videos for a content strategist. Reply with JSON only.", user, stream: false });
const parsed = parseJsonObject(content);
const items = Array.isArray(parsed) ? parsed : parsed?.videos ?? [];
const ids = new Set(batch.map(b => b.video_id));
db.transaction(() => { for (const it of items) if (ids.has(it.id) && Number.isFinite(Number(it.fit))) save.run(S, it.id, clamp01(Number(it.fit)), it.subject ?? null, it.packaging ?? null, it.note ?? null, model, nowIso()); })();
log(`judge ${Math.min(i + 10, top.length)}/${top.length}`);
} catch (e) { log(`judge batch failed (${e.message}); ranking continues on heuristic fit`); break; }
}
}
// ---------------------------------------------------------------- rank
function loadScored() {
const hitRows = db.prepare("SELECT video_id, COUNT(*) n, MIN(position) best FROM series_hits WHERE series_id=? GROUP BY video_id").all(S);
const hits = new Map(hitRows.map(r => [r.video_id, r]));
const channels = new Map(db.prepare("SELECT * FROM series_channels WHERE series_id=? AND status='ok'").all(S).map(c => [c.channel_key, c]));
const llmFit = new Map(db.prepare("SELECT video_id, fit FROM series_llm_fit WHERE series_id=?").all(S).map(r => [r.video_id, r.fit]));
const videos = db.prepare("SELECT * FROM series_videos WHERE series_id=? AND info_status='ok'").all(S).map(v => ({
...v, channel_key: channelKeyOf(v), query_hits: hits.get(v.video_id)?.n ?? 1, best_position: hits.get(v.video_id)?.best ?? null
}));
const scored = scoreCandidates(videos, channels, spec, { llmFit });
return { scored, channels };
}
function rank() {
const { scored } = loadScored();
const topN = num("top", spec.scoring.topN);
const selected = selectLearnFrom(scored, spec, { topN });
const lists = studyLists(scored, selected);
const runId = nowIso().replace(/[:.]/g, "-");
const insert = db.prepare(`INSERT INTO series_rankings(series_id,run_id,video_id,rank,selected,tier,learn_score,fit,performance,learnability,excluded,detail_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`);
const order = [...scored].sort((a, b) => (b.learn_score ?? -1) - (a.learn_score ?? -1));
const selIds = new Map(selected.map((r, i) => [r.video_id, i + 1]));
db.transaction(() => order.forEach((r, i) => {
const detail = { ...r, description: r.description?.slice(0, 600) ?? null };
insert.run(S, runId, r.video_id, r.learn_score == null ? null : i + 1, selIds.has(r.video_id) ? 1 : 0, r.tier, r.learn_score, r.fit, r.performance, r.learnability, r.excluded, JSON.stringify(detail));
}))();
log(`rank: run ${runId}: ${scored.length} scored, ${scored.filter(r => r.learn_score != null).length} eligible, ${selected.length} selected`);
selected.forEach((r, i) => console.log(` ${String(i + 1).padStart(2)}. ${r.learn_score.toFixed(3)} ${r.title} [${r.channel_name}, ${Number(r.views).toLocaleString()} views]`));
return { runId, scored, selected, lists };
}
// ---------------------------------------------------------------- transcripts
async function transcripts() {
const { fetchOriginalTranscript, formatTranscriptText } = await import("../transcript-fetch.js");
const run = db.prepare("SELECT run_id FROM series_rankings WHERE series_id=? ORDER BY run_id DESC LIMIT 1").get(S);
if (!run) throw new Error("No ranking yet. Run the rank command first.");
const want = num("top", budget.transcriptsTopN ?? 15);
const rows = db.prepare(`SELECT r.video_id, r.rank, r.detail_json FROM series_rankings r WHERE r.series_id=? AND r.run_id=? AND r.rank IS NOT NULL ORDER BY r.rank`).all(S, run.run_id);
const outDir = path.resolve("transcripts", "series-analysis", S);
fs.mkdirSync(outDir, { recursive: true });
const save = db.prepare("INSERT OR REPLACE INTO series_transcripts(series_id,video_id,status,error,language,segment_count,file_path,fetched_at) VALUES(?,?,?,?,?,?,?,?)");
const done = new Set(db.prepare("SELECT video_id FROM series_transcripts WHERE series_id=? AND status='ok' AND file_path IS NOT NULL").all(S).map(r => r.video_id));
let got = 0;
for (const row of rows) {
if (got >= want) break;
const d = JSON.parse(row.detail_json);
const file = path.join(outDir, `rank-${String(row.rank).padStart(3, "0")}_${row.video_id}.txt`);
if (done.has(row.video_id) && fs.existsSync(file)) { got++; continue; }
try {
const t = await fetchOriginalTranscript(row.video_id);
const n = (x, digits = 0) => x == null ? "n/a" : Number(x).toLocaleString("en-US", { maximumFractionDigits: digits });
const header = [
"=== VIDEO ===",
`Title: ${d.title}`, `URL: https://www.youtube.com/watch?v=${row.video_id}`, `Video ID: ${row.video_id}`,
`Channel: ${d.channel_name}`, `Published: ${d.published_at?.slice(0, 10) ?? "n/a"} (age ${n(d.age_days)} days)`,
`Duration: ${d.duration_seconds ? `${Math.floor(d.duration_seconds / 60)}m ${String(d.duration_seconds % 60).padStart(2, "0")}s` : "n/a"}`,
`Likes: ${n(d.likes)}`, `Description: ${(d.description ?? "").replace(/\s+/g, " ").slice(0, 1000) || "n/a"}`,
"", "=== PERFORMANCE ===",
`Learn-from rank: ${row.rank} (score ${n(d.learn_score, 3)}; tier ${d.tier})`,
`Views: ${n(d.views)}`, `Views per day: ${n(d.views_per_day, 1)}`,
`vs channel baseline: ${d.outlier_ratio == null ? "n/a" : `${n(d.outlier_ratio, 2)}x (${d.outlier_basis})`}`,
`Fit / performance / learnability: ${n(d.fit, 2)} / ${n(d.performance, 2)} / ${n(d.learnability, 2)}`,
"", "=== CHANNEL ===", `Subscribers: ${n(d.subscribers)}`, `Channel median views (recent): ${n(d.baseline_median_views)}`,
"", "=== TRANSCRIPT ===",
`Language: ${t.languageName} [${t.lang}] (detected via ${t.detection})`, `Fetched with: ${t.library}, ${t.segments.length} caption segments`, "", ""
].join("\n");
fs.writeFileSync(file, header + formatTranscriptText(t.segments) + "\n", "utf8");
save.run(S, row.video_id, "ok", null, t.lang, t.segments.length, file, nowIso());
got++;
log(`transcript ${got}/${want}: #${row.rank} ${d.title} [${t.lang}]`);
} catch (e) {
save.run(S, row.video_id, "error", String(e.message).slice(0, 400), null, null, null, nowIso());
log(` no transcript for #${row.rank} ${d.title}: ${e.message.slice(0, 120)}`);
}
await sleep(num("delay-ms", 1500));
}
log(`transcripts: ${got}/${want} saved in ${outDir}`);
}
// ---------------------------------------------------------------- study cohorts
/**
* Picks the study cohorts (new vs established, same rule, different age) and saves a verified transcript for each video
* in the layout the script-cohort-analysis skill reads. A transcript that fails the capture checks is replaced by the next candidate.
*/
async function cohorts() {
const { fetchOriginalTranscript, formatTranscriptText } = await import("../transcript-fetch.js");
const { selectCohortCandidates, checkTranscript } = await import("./cohorts.js");
const { scored } = loadScored();
const size = num("size", spec.cohorts.size);
const n = (x, d = 0) => x == null ? "n/a" : Number(x).toLocaleString("en-US", { maximumFractionDigits: d });
for (const which of ["new", "established"]) {
const outDir = path.resolve("transcripts", "series-analysis", S, `cohort-${which}`);
fs.mkdirSync(outDir, { recursive: true });
for (const f of fs.readdirSync(outDir)) if (/\.txt$/.test(f)) fs.rmSync(path.join(outDir, f));
const candidates = selectCohortCandidates(scored, spec, which);
log(`cohort ${which}: ${candidates.length} candidates pass the rule, taking ${size}`);
const chosen = [], rejected = [];
for (const r of candidates) {
if (chosen.length >= size) break;
try {
const t = await fetchOriginalTranscript(r.video_id);
const q = checkTranscript(t.segments, r.duration_seconds);
if (!q.ok) { rejected.push({ video_id: r.video_id, title: r.title, reason: q.problems.join("; ") }); log(` reject ${r.video_id} ${r.title.slice(0, 50)}: ${q.problems.join("; ")}`); continue; }
const idx = chosen.length + 1;
const header = [
"=== VIDEO ===",
`Title: ${r.title}`, `URL: https://www.youtube.com/watch?v=${r.video_id}`, `Video ID: ${r.video_id}`,
`Channel: ${r.channel_name}`, `Published: ${r.published_at?.slice(0, 10) ?? "n/a"} (age ${Math.round(r.age_days)} days)`, // no thousands separator: the cohort skill parses "age (\d+) days"
`Duration: ${Math.floor(r.duration_seconds / 60)}m ${String(r.duration_seconds % 60).padStart(2, "0")}s`,
`Category: ${r.category ?? "n/a"}`, `Likes: ${n(r.likes)}`,
`Description: ${(r.description ?? "").replace(/\s+/g, " ").slice(0, 600) || "n/a"}`,
"", "=== PERFORMANCE ===",
`Sample bucket: ${which.toUpperCase()} #${idx}`,
`Views: ${n(r.views)}`, `Views per day: ${n(r.views_per_day, 1)}`,
`Performance score: ${n(r.performance * 1000, 0)} (0-1000 blend of views vs the channel's typical upload, views per day and like rate)`,
`Outlier vs channel: ${r.outlier_ratio == null ? "n/a" : `${n(r.outlier_ratio, 2)}x (${r.outlier_basis})`}`,
"", "=== CHANNEL ===", `Subscribers: ${n(r.subscribers)}`, `Channel median views (recent uploads): ${n(r.baseline_median_views)}`,
"", "=== TRANSCRIPT ===",
`Language: ${t.languageName} [${t.lang}], original language (detected via ${t.detection}; tracks on video: ${t.availableTracks.join(", ")})`,
`Fetched with: ${t.library}, ${t.segments.length} caption segments, ${q.words} words, ${q.wpm.toFixed(0)} words/min, ${(q.coverage * 100).toFixed(0)}% of the video covered`, "", ""
].join("\n");
const file = path.join(outDir, `${which}-${String(idx).padStart(2, "0")}_${r.video_id}.txt`);
fs.writeFileSync(file, header + formatTranscriptText(t.segments) + "\n", "utf8");
chosen.push({ idx, video_id: r.video_id, title: r.title, channel: r.channel_name, subscribers: r.subscribers, age_days: Math.round(r.age_days), views: r.views,
performance: Math.round(r.performance * 1000), outlier_ratio: r.outlier_ratio, language: t.lang, words: q.words, wpm: Math.round(q.wpm), coverage: +q.coverage.toFixed(2), file: path.basename(file) });
log(` ${which} ${chosen.length}/${size}: ${r.title.slice(0, 60)} [${t.lang}, ${q.words} words, ${(q.coverage * 100).toFixed(0)}% covered]`);
} catch (e) {
rejected.push({ video_id: r.video_id, title: r.title, reason: `fetch failed: ${e.message.slice(0, 120)}` });
log(` skip ${r.video_id}: ${e.message.slice(0, 100)}`);
}
await sleep(num("delay-ms", 1500));
}
fs.writeFileSync(path.join(outDir, "_index.json"), JSON.stringify({ cohort: which, series: S, rule: spec.cohorts, generated_at: nowIso(), chosen, rejected }, null, 2), "utf8");
if (chosen.length < size) log(`WARNING: cohort ${which} has only ${chosen.length}/${size} verified transcripts`);
}
}
// ---------------------------------------------------------------- report
function report(ranked) {
const r = ranked ?? (() => {
const { scored } = loadScored();
const selected = selectLearnFrom(scored, spec);
return { runId: "adhoc", scored, selected, lists: studyLists(scored, selected) };
})();
const queries = db.prepare("SELECT tier, COUNT(*) n, SUM(status='done') done, SUM(result_count) results FROM series_queries WHERE series_id=? GROUP BY tier").all(S);
const coverage = {
candidates: db.prepare("SELECT COUNT(*) n FROM series_videos WHERE series_id=?").get(S).n,
enriched: db.prepare("SELECT COUNT(*) n FROM series_videos WHERE series_id=? AND info_status='ok'").get(S).n,
channels: db.prepare("SELECT COUNT(*) n FROM series_channels WHERE series_id=? AND status='ok'").get(S).n,
channelsWithBaseline: db.prepare("SELECT COUNT(*) n FROM series_channels WHERE series_id=? AND baseline_n>=5").get(S).n,
llmJudged: db.prepare("SELECT COUNT(*) n FROM series_llm_fit WHERE series_id=?").get(S).n
};
const transcriptRows = db.prepare("SELECT video_id, file_path FROM series_transcripts WHERE series_id=? AND status='ok'").all(S);
const md = buildReport({ spec, runId: r.runId, scored: r.scored, selected: r.selected, lists: r.lists, queries, coverage, transcripts: new Map(transcriptRows.map(t => [t.video_id, t.file_path])) });
const outDir = path.resolve("reports", "series-analysis");
fs.mkdirSync(outDir, { recursive: true });
const base = path.join(outDir, `${S}-${r.runId}`);
fs.writeFileSync(`${base}.md`, md, "utf8");
fs.writeFileSync(`${base}.json`, JSON.stringify({
series: S, run_id: r.runId, coverage, queries,
learn_from: r.selected.map(({ description, ...x }, i) => ({ rank: i + 1, ...x })),
breakouts: r.lists.breakouts.map(({ description, ...x }) => x),
contrast: r.lists.contrast.map(({ description, ...x }) => x)
}, null, 2), "utf8");
log(`report written: ${base}.md`);
}
// ---------------------------------------------------------------- dispatch
const stages = {
plan, search, mine, enrich, judge, transcripts, cohorts,
rank: async () => { rank(); },
report: async () => report(rank()),
all: async () => {
await plan(); await search(); mine(); await enrich();
if (args.llm) await judge();
const ranked = rank();
if (args.transcripts) await transcripts();
report(ranked);
}
};
if (!stages[command]) throw new Error(`Unknown command "${command}". Try: ${Object.keys(stages).join(", ")}`);
await stages[command]();
db.close();
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.