`src/topic-research/synthesize.js`

Code130 lines7.5 KBsrc/
import path from "node:path";
import { askJson, readJson, writeJson } from "./util.js";

const line = (f, body) => `${f.id} ${f.verified ? "" : "[unverified] "}${f.certainty && f.certainty !== "stated" ? `[${f.certainty}${f.attributed_to ? `: ${f.attributed_to}` : ""}] ` : ""}${body}`;

const FACT_LINES = {
  events: f => line(f, `DATE=${f.date || "unknown"} | ${f.what}${f.who ? ` (who: ${f.who})` : ""}`),
  figures: f => line(f, `METRIC=${f.metric} | VALUE=${f.unit && !String(f.value).toLowerCase().includes(String(f.unit).toLowerCase()) ? `${f.value} ${f.unit}` : f.value} | AS_OF=${f.as_of || "?"} | ${f.context || ""}`),
  surprises: f => line(f, `${f.point} -- ${f.why_surprising || ""}`),
  causes: f => line(f, `[${f.kind}] ${f.factor}: ${f.explanation || ""}`)
};

const COMMON = `RULES
- Merge facts that describe the same thing into ONE item and list every supporting fact id in "fact_ids". Never invent a fact id; never add information that is not in the listed facts.
- If supporting facts disagree (different dates, numbers, causes), keep the item, list all ids, and describe the disagreement in "conflict" (otherwise "").
- Facts tagged [hedged], [attributed] or [unverified] are weaker; say so in "note" when an item rests only on them.
- Keep numbers exactly as written in the facts, with their units.
- Return ONE JSON object.`;

const PROMPTS = {
  events: (topic, facts) => `Topic: "${topic}". Below are dated events extracted from several sources. Build ONE chronological timeline of the topic.
${COMMON}
Schema: {"timeline": [{"date": "as best known e.g. 2008 or Mar 2013", "sort_key": "YYYY[-MM[-DD]]", "event": "one sentence", "significance": "why it matters to the story, 1 sentence, only if the facts say so", "conflict": "", "note": "", "fact_ids": []}]}
Order by sort_key. Include every distinct event; do not drop minor dated ones.

FACTS
${facts}`,

  figures: (topic, facts) => `Topic: "${topic}". Below are hard numbers extracted from several sources. Build the consolidated table of figures.
${COMMON}
Schema: {"figures": [{"metric": "", "value": "exactly as stated with unit", "as_of": "year/period or unknown", "context": "one line", "conflict": "e.g. S1 says 5,000 crore, S3 says 4,500 crore", "note": "", "fact_ids": []}]}
MERGE facts into ONE item (listing all their fact ids) whenever they report the same measure for the same thing and period, even if worded differently or in another language: e.g. three sources saying PNB alleged fraud of 787 crore = ONE item with three fact ids; four sources saying 150 million dollars of FCCBs = ONE item. Keep SEPARATE only facts that differ in scope or period (peak global share vs India share vs one product; different years; partial vs total), putting the scope in "metric". Use "conflict" only when sources give different values for the same measure and period; write both values. "value" must be a clean self-contained figure with its unit once (e.g. "354.51 crore rupees", "80%"), translated to English.

FACTS
${facts}`,

  surprises: (topic, facts) => `Topic: "${topic}". Below are points sources framed as shocking, surprising or counter-intuitive. Pick the strongest for a video: ones a viewer would not expect, that are concrete, and that more than one source supports rank higher.
${COMMON}
Schema: {"surprises": [{"point": "one punchy sentence", "why_surprising": "", "strength": "high|medium|low", "note": "", "conflict": "", "fact_ids": []}]}
Order strongest first.

FACTS
${facts}`,

  causes: (topic, facts) => `Topic: "${topic}". Below are causes and turning points extracted from several sources. Consolidate them into the distinct reasons the story unfolded as it did.
${COMMON}
Schema: {"causes": [{"factor": "short name", "kind": "strategic|financial|market|technology|governance|external|other", "explanation": "2 sentences max, from the facts", "conflict": "", "note": "", "fact_ids": []}]}
Order by how many independent sources support them.

FACTS
${facts}`
};

function attach(items, factIndex) {
  const out = [];
  for (const item of items ?? []) {
    const ids = [...new Set((item.fact_ids ?? []).filter(id => factIndex.has(id)))];
    if (!ids.length) continue; // ungrounded: dropped
    const facts = ids.map(id => factIndex.get(id));
    const docs = [...new Set(facts.map(f => f.doc))];
    out.push({
      ...item,
      fact_ids: ids,
      source_count: docs.length,
      verified_quotes: facts.filter(f => f.verified).length,
      evidence: facts.map(f => ({ id: f.id, doc: f.doc, t: f.seconds, verified: f.verified, quote: f.quote }))
    });
  }
  return out;
}

export async function synthesize({ topic, docs, extractions, dir, llmCache }) {
  const usable = extractions.filter(e => e.relevance?.about_topic !== false && e.relevance?.share !== "little");
  const factIndex = new Map();
  const grouped = { events: [], figures: [], surprises: [], causes: [] };
  for (const e of usable) {
    for (const kind of Object.keys(grouped)) {
      for (const f of e[kind]) { factIndex.set(f.id, { ...f, doc: e.key }); grouped[kind].push(f); }
    }
  }

  const sections = {};
  for (const kind of Object.keys(grouped)) {
    const facts = grouped[kind].map(FACT_LINES[kind]).join("\n");
    const out = await askJson(PROMPTS[kind](topic, facts), { cacheDir: llmCache, label: `synth ${kind}` });
    const key = { events: "timeline" }[kind] ?? kind;
    sections[key] = attach(out[key], factIndex);
  }
  // Deterministic order, whatever the model returned.
  sections.timeline.sort((a, b) => String(a.sort_key || "9999").localeCompare(String(b.sort_key || "9999")) || a.fact_ids[0].localeCompare(b.fact_ids[0]));

  const digest = usable.map(e => `${e.key}: ${e.summary}`).join("\n");
  const compact = k => sections[k].map(i => `- ${i.event ?? i.metric ?? i.point ?? i.factor}: ${i.value ?? i.explanation ?? i.why_surprising ?? i.significance ?? ""} (${i.source_count} source${i.source_count > 1 ? "s" : ""})`).join("\n");
  const flagged = usable.flatMap(e => e.claims_to_verify.map(c => `${e.key}: ${c.claim} (${c.why})`)).join("\n");
  const entities = usable.flatMap(e => e.entities.map(x => ({ ...x, doc: e.key })));

  const overviewPrompt = `Topic: "${topic}". You have research notes from ${usable.length} sources. Write the topic briefing using ONLY the notes below.
Return ONE JSON object:
{"overview": "5-8 sentence plain-English account of the whole story, start to end",
 "one_line": "single sentence thesis of the story",
 "entities": [{"name": "", "type": "person|company|product|institution|place", "role": "one line from the notes"}],
 "open_questions": [{"question": "what a viewer will ask that the sources do not settle", "why": ""}],
 "contested": [{"claim": "", "why_contested": ""}],
 "verification_queries": ["web search queries that would confirm the single-source figures, contested claims and gaps, written as plain questions"]}
Merge duplicate entities (spelling variants). Aim for 6-12 verification queries focused on hard numbers and dates.

SOURCE SUMMARIES
${digest}

TIMELINE
${compact("timeline")}

FIGURES
${compact("figures")}

CAUSES
${compact("causes")}

CLAIMS SOURCES FLAGGED
${flagged || "(none)"}

ENTITIES MENTIONED
${[...new Set(entities.map(e => `${e.name} (${e.type}): ${e.role}`))].join("\n")}`;
  const brief = await askJson(overviewPrompt, { cacheDir: llmCache, label: "synth overview" });

  const result = { topic, generatedAt: new Date().toISOString(), sourcesUsed: usable.map(e => e.key), sourcesExcluded: extractions.filter(e => !usable.includes(e)).map(e => ({ key: e.key, note: e.relevance?.note })), ...brief, ...sections };
  writeJson(path.join(dir, "synthesis.json"), result);
  return result;
}
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.