Docs/Skills and code/src/

src/topic-research/extract.js

Code113 lines5.6 KBsrc/

import path from "node:path";
import { askJson, norm, parseStamp, stamp, readJson, writeJson } from "./util.js";

const CHUNK_WORDS = 6000;

/** Groups caption segments into ~30s timestamped paragraphs: [{start, text}]. */
export function toParagraphs(segments, seconds = 30) {
  const out = [];
  let cur = null;
  for (const seg of segments) {
    if (!cur || seg.start - cur.start >= seconds) {
      cur = { start: seg.start, parts: [] };
      out.push(cur);
    }
    cur.parts.push(seg.text);
  }
  return out.map(p => ({ start: p.start, text: p.parts.join(" ").replace(/\s+/g, " ").trim() }));
}

export function chunkParagraphs(paragraphs, maxWords = CHUNK_WORDS) {
  const chunks = [];
  let cur = [], words = 0;
  for (const p of paragraphs) {
    const n = p.text.split(/\s+/).length;
    if (cur.length && words + n > maxWords) { chunks.push(cur); cur = []; words = 0; }
    cur.push(p);
    words += n;
  }
  if (cur.length) chunks.push(cur);
  return chunks;
}

const render = ps => ps.map(p => `[${stamp(p.start)}] ${p.text}`).join("\n\n");

export function buildPrompt({ topic, doc, chunk, index, total }) {
  return `You are extracting research notes for a documentary-style video about: "${topic}".
You are reading ONE source: "${doc.meta.title}" by ${doc.meta.channel}. It is an auto-captioned transcript${doc.lang ? ` (language code: ${doc.lang})` : ""}${total > 1 ? `, part ${index + 1} of ${total}` : ""}. It may be in Hindi, Hinglish or another language and may mis-spell names.

RULES
- Use ONLY what this transcript says. Never add outside knowledge or guess.
- Write every field in English, but "quote" must be copied EXACTLY (verbatim, contiguous, max 25 words) from the transcript in its original language and script.
- "t" is the [mm:ss] stamp of the paragraph the quote sits in.
- Fix obvious name mis-spellings (e.g. "Moser Bear" -> "Moser Baer").
- Record numbers exactly as spoken, keeping the speaker's unit ("crore", "lakh", "million", "%"). Do not convert, round or combine numbers.
- certainty: "stated" if said plainly, "hedged" if the speaker hedges (allegedly, I think, around, reportedly), "attributed" if the speaker credits a report, person or news item (put that in "attributed_to").
- date: as stated ("2008", "mid-2000s", "after the crisis"); "unknown" if none. sort_key: best-effort "YYYY", "YYYY-MM" or "YYYY-MM-DD" ("" if undatable).
- Skip filler, ads, sponsor reads, channel promos. Include every distinct date, number and cause you find: completeness matters.

Return ONE JSON object with exactly these keys:
{
  "relevance": {"about_topic": true|false, "share": "most"|"part"|"little", "note": "one line"},
  "summary": "3-4 sentence summary of what the source says about the topic",
  "events": [{"date": "", "sort_key": "", "what": "", "who": "", "certainty": "", "attributed_to": "", "quote": "", "t": "mm:ss"}],
  "figures": [{"metric": "what is measured", "value": "as spoken", "unit": "", "as_of": "", "context": "", "certainty": "", "attributed_to": "", "quote": "", "t": "mm:ss"}],
  "surprises": [{"point": "", "why_surprising": "", "quote": "", "t": "mm:ss"}],
  "causes": [{"factor": "", "kind": "strategic|financial|market|technology|governance|external|other", "explanation": "", "quote": "", "t": "mm:ss"}],
  "entities": [{"name": "", "type": "person|company|product|institution|place", "role": ""}],
  "claims_to_verify": [{"claim": "", "why": "hedged | contradicts common knowledge | promotional | vague"}]
}

TRANSCRIPT
${render(chunk)}`;
}

const KINDS = { events: "e", figures: "f", surprises: "s", causes: "c" };

/** Checks each quote against the transcript text and pins it to the paragraph it appears in. */
export function verifyQuotes(items, paragraphs) {
  const normParas = paragraphs.map(p => norm(p.text));
  for (const item of items) {
    const q = norm(item.quote);
    item.verified = false;
    if (q.split(" ").length >= 3) {
      const idx = normParas.findIndex(p => p.includes(q));
      if (idx >= 0) { item.verified = true; item.seconds = paragraphs[idx].start; }
      else {
        // Quote may straddle two paragraphs.
        const j = normParas.findIndex((p, i) => i + 1 < normParas.length && (p + " " + normParas[i + 1]).includes(q));
        if (j >= 0) { item.verified = true; item.seconds = paragraphs[j].start; }
      }
    }
    if (item.seconds == null) item.seconds = parseStamp(item.t);
  }
}

export async function extractDocument({ topic, doc, dir, llmCache }) {
  const file = path.join(dir, "extractions", `${doc.meta.video_id ?? doc.key}.json`);
  const cached = readJson(file);
  if (cached) return cached;

  const paragraphs = toParagraphs(doc.segments);
  const chunks = chunkParagraphs(paragraphs);
  const merged = { key: doc.key, relevance: null, summary: [], events: [], figures: [], surprises: [], causes: [], entities: [], claims_to_verify: [] };

  for (const [index, chunk] of chunks.entries()) {
    const out = await askJson(buildPrompt({ topic, doc, chunk, index, total: chunks.length }), { cacheDir: llmCache, label: `extract ${doc.key}` });
    merged.relevance ??= out.relevance ?? null;
    if (out.summary) merged.summary.push(String(out.summary));
    for (const k of [...Object.keys(KINDS), "entities", "claims_to_verify"]) {
      if (Array.isArray(out[k])) merged[k].push(...out[k]);
    }
  }
  merged.summary = merged.summary.join(" ");

  for (const [kind, letter] of Object.entries(KINDS)) {
    verifyQuotes(merged[kind], paragraphs);
    merged[kind].forEach((item, i) => { item.id = `${doc.key}-${letter}${i + 1}`; });
  }
  merged.stats = Object.fromEntries(Object.keys(KINDS).map(k => [k, { total: merged[k].length, verified: merged[k].filter(x => x.verified).length }]));
  writeJson(file, merged);
  return merged;
}

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.