Docs/Skills and code/src/

src/topic-research/sources/youtube.js

Code124 lines5.5 KBsrc/

import path from "node:path";
import fs from "node:fs";
import { searchVideos } from "../../youtube.js";
import { fetchOriginalTranscript, formatTranscriptText } from "../../transcript-fetch.js";
import { squash, readJson, writeJson } from "../util.js";

// Fixed query templates: the same topic always produces the same searches.
export const QUERY_TEMPLATES = [
  "{topic}",
  "{entity}",
  "{entity} documentary",
  "{entity} explained",
  "{entity} what happened",
  "{entity} case study",
  "{entity} rise and fall",
  "{entity} collapse scandal"
];

// Words that frame a topic as a story but are not part of the subject itself.
const FRAMING = new Set(["the", "a", "an", "of", "fall", "rise", "downfall", "collapse", "story", "death", "end", "how", "why", "what", "happened", "to", "and", "decline", "failure", "demise", "saga"]);

export function deriveEntity(topic) {
  const words = String(topic).split(/\s+/).filter(w => !FRAMING.has(w.toLowerCase().replace(/[^a-z]/g, "")));
  return words.join(" ").trim() || topic;
}

const NOISE = /for sale|price in|advertis|tvc\b|commercial|\bident\b|trailer|unboxing|ringtone|lyrics|\bsong\b|full movie/i;
const FRAMING_TITLE = /fall|downfall|fail|collaps|rise|story|case study|what happened|why|scandal|documentary|explained|went wrong|shocking|truth|bankrupt|debt|crash|death/i;

/** Why a search hit is dropped before ranking, or null if it passes the topic/noise filter. */
export function rejectReason(v, entityKey) {
  if (!squash(v.title).includes(entityKey) && !squash(v.channel_name).includes(entityKey)) return "entity_not_in_title_or_channel";
  if (NOISE.test(v.title)) return "noise_title";
  if (!v.duration_seconds || v.duration_seconds < 180) return "under_3_minutes";
  return null;
}

export function scoreCandidate(v, entityKey) {
  if (rejectReason(v, entityKey)) return null;
  const titleHit = squash(v.title).includes(entityKey);
  const minutes = v.duration_seconds / 60;
  return (
    Math.log10((v.view_count ?? 0) + 10) +
    Math.min(v.found_by.length, 4) * 0.5 +
    (minutes >= 5 && minutes <= 60 ? 1 : 0) +
    (FRAMING_TITLE.test(v.title) ? 1.5 : 0) +
    (titleHit ? 1 : 0)
  );
}

export async function searchSources({ topic, entity, dir, maxVideos = 8, queryLimit = 20, extraQueries = [], refresh = false }) {
  const file = path.join(dir, "search.json");
  const cached = !refresh && readJson(file);
  if (cached) return cached;

  const queries = [...new Set([...QUERY_TEMPLATES.map(t => t.replace("{topic}", topic).replace("{entity}", entity)), ...extraQueries])];
  const byId = new Map();
  for (const query of queries) {
    let found = [];
    try { found = await searchVideos(query, queryLimit); } catch (e) { console.warn(`[search] "${query}" failed: ${e.message}`); }
    for (const v of found) {
      const row = byId.get(v.video_id) ?? { ...v, found_by: [] };
      row.found_by.push(query);
      byId.set(v.video_id, row);
    }
  }

  const entityKey = squash(entity);
  const scored = [...byId.values()]
    .map(v => ({ v, score: scoreCandidate(v, entityKey) }))
    .filter(x => x.score != null)
    .sort((a, b) => b.score - a.score || a.v.video_id.localeCompare(b.v.video_id));

  const kept = new Set(scored.map(x => x.v.video_id));
  const result = {
    topic, entity, queries, searchedAt: new Date().toISOString(),
    candidatesSeen: byId.size, candidatesKept: scored.length,
    selected: scored.slice(0, maxVideos).map(({ v, score }) => ({
      video_id: v.video_id, title: v.title, channel: v.channel_name, url: v.url,
      views: v.view_count, duration_seconds: v.duration_seconds, published: v.published_at_text,
      found_by: v.found_by.length, score: +score.toFixed(3)
    })),
    rejected: [...byId.values()].filter(v => !kept.has(v.video_id))
      .map(v => ({ video_id: v.video_id, title: v.title, views: v.view_count, duration_seconds: v.duration_seconds })),
    // Every video seen, with channel details, so the full discovery set can be exported.
    candidates: [...byId.values()].map(v => {
      const rank = scored.findIndex(x => x.v.video_id === v.video_id);
      return {
        video_id: v.video_id, title: v.title, url: v.url, channel: v.channel_name, channel_url: v.channel_url,
        views: v.view_count, published: v.published_at_text, duration_seconds: v.duration_seconds,
        found_by: v.found_by, reject_reason: rejectReason(v, entityKey), rank: rank >= 0 ? rank + 1 : null,
        selected: rank >= 0 && rank < maxVideos
      };
    })
  };
  writeJson(file, result);
  return result;
}

/** Fetches (and caches) transcripts, returning source documents in the shape extraction expects. */
export async function collectDocuments(selected, dir) {
  const docs = [];
  for (const [i, v] of selected.entries()) {
    const cache = path.join(dir, "transcripts", `${v.video_id}.json`);
    let t = readJson(cache);
    if (!t) {
      try {
        t = await fetchOriginalTranscript(v.video_id);
        writeJson(cache, t);
        fs.writeFileSync(cache.replace(/\.json$/, ".txt"), formatTranscriptText(t.segments));
      } catch (error) {
        console.warn(`[transcript] ${v.video_id} skipped: ${String(error.message).slice(0, 160)}`);
        docs.push({ key: `S${i + 1}`, kind: "youtube", meta: v, error: error.message, segments: [] });
        continue;
      }
    }
    docs.push({ key: `S${i + 1}`, kind: "youtube", meta: v, lang: t.lang, segments: t.segments });
  }
  return docs;
}

export const sourceUrl = (doc, seconds) =>
  `https://www.youtube.com/watch?v=${doc.meta.video_id}${seconds != null ? `&t=${Math.floor(seconds)}s` : ""}`;

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.