Docs/Skills and code/src/

src/stages/05-expand-discovery.js

Code195 lines5.8 KBsrc/

import { config } from "../../config.js";
import { getDb } from "../db.js";
import { searchVideos } from "../youtube.js";
import { ensureAnalysisSchema } from "../analysis-schema.js";

const db = getDb();
ensureAnalysisSchema();

const limitArg = process.argv.find(value => value.startsWith("--limit="));
const limit = limitArg ? Number(limitArg.split("=")[1]) : 10;

const candidates = db.prepare(`
  SELECT
    c.id AS concept_id,
    c.canonical_name,
    c.concept_type,
    COUNT(DISTINCT cc.channel_url) AS channel_count,
    COUNT(DISTINCT vc.video_id) AS video_count,
    COALESCE(MAX(v.view_count), 0) AS max_views
  FROM concepts c
  LEFT JOIN channel_concepts cc ON cc.concept_id = c.id
  LEFT JOIN video_concepts vc ON vc.concept_id = c.id
  LEFT JOIN videos v ON v.video_id = vc.video_id
  WHERE c.ecosystem = ?
    AND c.concept_type IN ('entity', 'theme', 'angle', 'subject', 'idea')
    AND NOT EXISTS (
      SELECT 1
      FROM searches s
      WHERE lower(trim(s.query)) = lower(trim(c.canonical_name))
    )
  GROUP BY c.id
  ORDER BY
    channel_count DESC,
    video_count DESC,
    max_views DESC
  LIMIT ?
`).all(config.ecosystem.name, limit);

const saveCandidate = db.prepare(`
  INSERT INTO discovery_candidates (
    ecosystem, concept_id, query, source_type, source_ref,
    evidence_json, priority, status, created_at
  ) VALUES (?, ?, ?, ?, ?, ?, ?, 'candidate', ?)
  ON CONFLICT(ecosystem, query) DO UPDATE SET
    concept_id = excluded.concept_id,
    evidence_json = excluded.evidence_json,
    priority = excluded.priority
`);

const saveSearch = db.prepare(`
  INSERT INTO searches (query, searched_at)
  VALUES (?, ?)
  ON CONFLICT(query) DO UPDATE SET searched_at = excluded.searched_at
`);

const getSearch = db.prepare(`
  SELECT id FROM searches WHERE query = ?
`);

const saveOrigin = db.prepare(`
  INSERT OR REPLACE INTO search_origins (
    search_id, ecosystem, concept_id, source_type, source_ref, created_at
  ) VALUES (?, ?, ?, ?, ?, ?)
`);

const insertChannel = db.prepare(`
  INSERT INTO channels (channel_url, name, discovered_at)
  VALUES (?, ?, ?)
  ON CONFLICT(channel_url) DO UPDATE SET name = excluded.name
`);

const insertVideo = db.prepare(`
  INSERT INTO videos (
    video_id, title, url, channel_url, channel_name, thumbnail_url,
    duration, view_count, published_at, discovered_at
  ) VALUES (
    @video_id, @title, @url, @channel_url, @channel_name, @thumbnail_url,
    @duration, @view_count, @published_at, @discovered_at
  )
  ON CONFLICT(video_id) DO UPDATE SET
    title = excluded.title,
    url = excluded.url,
    channel_url = COALESCE(videos.channel_url, excluded.channel_url),
    channel_name = COALESCE(videos.channel_name, excluded.channel_name),
    thumbnail_url = COALESCE(videos.thumbnail_url, excluded.thumbnail_url),
    duration = COALESCE(videos.duration, excluded.duration),
    view_count = COALESCE(videos.view_count, excluded.view_count),
    published_at = COALESCE(videos.published_at, excluded.published_at)
`);

const insertVideoSearch = db.prepare(`
  INSERT OR IGNORE INTO video_searches (video_id, search_id, position)
  VALUES (?, ?, ?)
`);

function saveResults(searchId, videos) {
  const timestamp = new Date().toISOString();

  db.transaction(() => {
    videos.forEach((video, index) => {
      if (video.channel_url && video.channel_name) {
        insertChannel.run(video.channel_url, video.channel_name, timestamp);
      }

      insertVideo.run({ ...video, discovered_at: timestamp });
      insertVideoSearch.run(video.video_id, searchId, index + 1);
    });
  })();
}

let searched = 0;
let failed = 0;

for (const candidate of candidates) {
  const query = candidate.canonical_name.trim();
  const evidence = {
    conceptType: candidate.concept_type,
    channels: candidate.channel_count,
    videos: candidate.video_count,
    maxViews: candidate.max_views
  };

  const priority =
    candidate.channel_count * 4 +
    candidate.video_count * 2 +
    Math.log10(Math.max(candidate.max_views, 1));

  saveCandidate.run(
    config.ecosystem.name,
    candidate.concept_id,
    query,
    "vocabulary",
    String(candidate.concept_id),
    JSON.stringify(evidence),
    priority,
    new Date().toISOString()
  );

  console.log(`\n=== Recursive discovery ===`);
  console.log(`Vocabulary: ${candidate.concept_type} :: ${query}`);
  console.log("Evidence:", evidence);

  try {
    const searchedAt = new Date().toISOString();
    saveSearch.run(query, searchedAt);
    const search = getSearch.get(query);

    if (!search) {
      throw new Error(`Search record was not created for "${query}".`);
    }

    saveOrigin.run(
      search.id,
      config.ecosystem.name,
      candidate.concept_id,
      "vocabulary",
      String(candidate.concept_id),
      searchedAt
    );

    const videos = await searchVideos(query, config.youtube.resultsPerSearch);
    saveResults(search.id, videos);

    db.prepare(`
      UPDATE discovery_candidates
      SET status = 'searched', searched_at = ?
      WHERE ecosystem = ? AND query = ?
    `).run(searchedAt, config.ecosystem.name, query);

    console.log(`Saved ${videos.length} videos from recursive search.`);
    searched++;
  } catch (error) {
    failed++;
    const message = error instanceof Error ? error.message : String(error);

    db.prepare(`
      UPDATE discovery_candidates
      SET status = 'failed'
      WHERE ecosystem = ? AND query = ?
    `).run(config.ecosystem.name, query);

    console.error("Recursive search failed:", { query, error: message });
  }
}

console.log("\n=== Recursive discovery summary ===");
console.log({
  candidatesConsidered: candidates.length,
  searched,
  failed,
  totalSearches: db.prepare("SELECT COUNT(*) AS count FROM searches").get().count,
  totalVideos: db.prepare("SELECT COUNT(*) AS count FROM videos").get().count,
  totalChannels: db.prepare("SELECT COUNT(*) AS count FROM channels").get().count
});

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.