Docs/Skills and code/src/

src/topic-research/export-csv.js

Code47 lines2.3 KBsrc/

Mentioned on: page 12

#!/usr/bin/env node
// Usage: node src/topic-research/export-csv.js <search.json dir> [--resubscribe]
// Writes <dir>/all-videos.csv: every video the search saw, with channel subscriber counts.
// Needs a search.json that has `candidates` (written by searchSources).
import fs from "node:fs";
import path from "node:path";
import { Innertube, Log } from "youtubei.js";
import { readJson } from "./util.js";

Log.setLevel(Log.Level.NONE);
const dir = process.argv[2];
const search = readJson(path.join(dir, "search.json"));
if (!search?.candidates) { console.error("search.json has no `candidates`; re-run the search first."); process.exit(1); }

const yt = await Innertube.create();
const subsCache = new Map();
async function channelInfo(channelUrl) {
  if (!channelUrl) return { id: "", subs: "" };
  if (subsCache.has(channelUrl)) return subsCache.get(channelUrl);
  let out = { id: "", subs: "" };
  try {
    const endpoint = await yt.resolveURL(channelUrl);
    const id = endpoint?.payload?.browseId ?? "";
    const channel = await yt.getChannel(id);
    const text = JSON.stringify(channel.header).match(/"text":"([\d.,]+[KMB]? subscribers?)"/i)?.[1] ?? "";
    out = { id, subs: text.replace(/ subscribers?/i, "") };
  } catch (error) {
    console.warn(`[subs] ${channelUrl}: ${String(error.message).slice(0, 80)}`);
  }
  subsCache.set(channelUrl, out);
  return out;
}

const q = v => `"${String(v ?? "").replace(/"/g, '""')}"`;
const header = ["video_id", "title", "url", "channel", "channel_id", "channel_url", "channel_subscribers", "views", "age", "duration_seconds",
  "found_by_queries", "passes_filter", "reject_reason", "selected_for_research", "transcript_extracted"];
const rows = [header];
for (const c of search.candidates) {
  const ch = await channelInfo(c.channel_url);
  const transcript = fs.existsSync(path.join(dir, "transcripts", `${c.video_id}.json`));
  rows.push([c.video_id, c.title, c.url, c.channel, ch.id, c.channel_url, ch.subs, c.views, c.published, c.duration_seconds,
    c.found_by.length, !c.reject_reason, c.reject_reason ?? "", c.selected, c.selected ? transcript : "not_attempted"]);
}
const out = path.join(dir, "all-videos.csv");
fs.writeFileSync(out, "" + rows.map(r => r.map(q).join(",")).join("\n") + "\n");
console.log(`${rows.length - 1} videos, ${subsCache.size} channels -> ${out}`);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.