Docs/Skills and code/src/
src/stages/26-discovery-v2.js
import { filterQualifiedRows } from "../analysis/qualification-scope.js";
import fs from 'node:fs';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { config } from '../../config.js';
import { getDb } from '../db.js';
import { searchVideos } from '../youtube.js';
import { callLLM, parseJsonObject } from '../llm.js';
import { ensureDiscoveryV2Schema } from '../discovery/v2-schema.js';
import { ensureAnalysisSchema } from '../analysis-schema.js';
import { parseCount } from '../metadata-utils.js';
import { parseArgs, normalizeQuery, uniqueQueries, videoToChannelUrl } from '../discovery/v2-utils.js';
const args = parseArgs(process.argv.slice(2));
const here = path.dirname(fileURLToPath(import.meta.url));
const seedPath = config.discoverySeedPath;
const seedSettings = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
const seeds = { ...seedSettings, ...(config.discovery ?? {}), topicSeeds: config.ecosystem.discovery?.topicSeeds ?? seedSettings.topicSeeds ?? [], adjacentSeeds: config.ecosystem.discovery?.adjacentSeeds ?? seedSettings.adjacentSeeds ?? [], routeSettings: config.discovery?.routes ?? seedSettings.routeSettings, search: { ...seedSettings.search, resultsPerQuery: config.discovery?.resultsPerQuery ?? seedSettings.search?.resultsPerQuery, delayMs: config.discovery?.delayMs ?? seedSettings.search?.delayMs }, llm: { ...seedSettings.llm, ...config.discovery?.llm, maxHypothesesPerRoute: config.discovery?.maxHypothesesPerRoute ?? seedSettings.llm?.maxHypothesesPerRoute } };
const db = ensureDiscoveryV2Schema();
ensureAnalysisSchema();
const phaseId = String(args.phase || new Date().toISOString().replace(/[:.]/g, '-'));
const routes = String(args.routes || 'A,B,C,D').split(',').map(x => x.trim().toUpperCase()).filter(x => ['A','B','C','D'].includes(x));
const limitOverride = Number(args.queries || 0);
const resultsPerQuery = Math.max(5, Math.min(50, Number(args.results || seeds.search?.resultsPerQuery || 25)));
const useLLM = Boolean(args.llm);
const delayMs = Math.max(0, Number(args.delay ?? seeds.search?.delayMs ?? 350));
if (!routes.length) throw new Error('No valid routes. Use --routes=A,B,C,D');
const now = () => new Date().toISOString();
const getChannels = db.prepare(`SELECT c.channel_url,c.name,c.subscriber_count,c.video_count,COALESCE((SELECT qlr.classification FROM channel_qualification_llm_results qlr WHERE qlr.channel_url=c.channel_url AND qlr.processing_status='succeeded' ORDER BY qlr.created_at DESC LIMIT 1),cq.classification,'unknown') classification,COALESCE(vv.observed_videos,0) observed_videos,COALESCE(vv.max_views,0) max_views FROM channels c LEFT JOIN channel_qualifications cq ON cq.channel_url=c.channel_url LEFT JOIN (SELECT channel_url,COUNT(*) observed_videos,MAX(COALESCE(view_count,0)) max_views FROM videos WHERE channel_url IS NOT NULL GROUP BY channel_url) vv ON vv.channel_url=c.channel_url ORDER BY CASE COALESCE((SELECT qlr.classification FROM channel_qualification_llm_results qlr WHERE qlr.channel_url=c.channel_url AND qlr.processing_status='succeeded' ORDER BY qlr.created_at DESC LIMIT 1),cq.classification,'unknown') WHEN 'core' THEN 0 WHEN 'adjacent' THEN 1 WHEN 'unknown' THEN 2 ELSE 3 END,COALESCE(vv.observed_videos,0) DESC,COALESCE(vv.max_views,0) DESC,c.name COLLATE NOCASE`);
const channelRows = getChannels.all();
const channelRowsEligible = filterQualifiedRows(db, channelRows, "channel", { label: "26-discovery-v2.js" });
channelRows.splice(0, channelRows.length, ...channelRowsEligible);
const channelByUrl = new Map(channelRows.map(r => [r.channel_url, r]));
const videoRows = db.prepare(`SELECT v.video_id,v.title,v.view_count,v.channel_url,v.channel_name FROM videos v WHERE COALESCE(v.title,'')<>'' ORDER BY COALESCE(v.view_count,0) DESC`).all();
const videoRowsEligible = filterQualifiedRows(db, videoRows, "video", { label: "26-discovery-v2.js" });
videoRows.splice(0, videoRows.length, ...videoRowsEligible);
const insertPhase = db.prepare(`INSERT INTO discovery_v2_phases(phase_id,started_at,routes_json,status,settings_json) VALUES(?,?,?,'running',?)`);
const saveQuery = db.prepare(`INSERT OR IGNORE INTO discovery_v2_queries(phase_id,route,query,source_type,source_ref,hypothesis,status,created_at) VALUES(?,?,?,?,?,?,'pending',?)`);
const updateQuery = db.prepare(`UPDATE discovery_v2_queries SET status=?,result_count=?,error=?,finished_at=? WHERE phase_id=? AND route=? AND query=?`);
const saveChannel = db.prepare(`INSERT INTO channels(channel_url,name,discovered_at) VALUES(?,?,?) ON CONFLICT(channel_url) DO UPDATE SET name=CASE WHEN channels.name IS NULL OR trim(channels.name)='' THEN excluded.name ELSE channels.name END`);
const saveVideo = db.prepare(`INSERT INTO videos(video_id,title,url,channel_url,channel_name,thumbnail_url,duration,view_count,published_at,discovered_at) VALUES(@video_id,@title,@url,@channel_url,@channel_name,@thumbnail_url,@duration,@view_count,@published_at,@discovered_at) ON CONFLICT(video_id) DO UPDATE SET title=COALESCE(NULLIF(videos.title,''),excluded.title),url=COALESCE(videos.url,excluded.url),channel_url=COALESCE(videos.channel_url,excluded.channel_url),channel_name=COALESCE(videos.channel_name,excluded.channel_name),thumbnail_url=COALESCE(videos.thumbnail_url,excluded.thumbnail_url),duration=COALESCE(videos.duration,excluded.duration),view_count=COALESCE(videos.view_count,excluded.view_count),published_at=COALESCE(videos.published_at,excluded.published_at)`);
const saveVideoSearch = db.prepare(`INSERT OR IGNORE INTO video_searches(video_id,search_id,position) VALUES(?,?,?)`);
const saveEvent = db.prepare(`INSERT OR IGNORE INTO discovery_v2_channel_events(phase_id,channel_url,channel_name,route,query,first_video_id,result_position,was_new_channel,first_seen_at) VALUES(?,?,?,?,?,?,?,?,?)`);
const saveSearch = db.prepare(`INSERT INTO searches(query,searched_at) VALUES(?,?) ON CONFLICT(query) DO UPDATE SET searched_at=excluded.searched_at`);
const getSearch = db.prepare(`SELECT id FROM searches WHERE query=?`);
function extractTopics(title) {
const clean = String(title || '').replace(/\[[^\]]*\]|\([^)]*\)/g,' ').replace(/\b(official|full documentary|documentary|explained|analysis|4k|hd|shorts|episode|part \d+)\b/ig,' ').replace(/[|:—–]/g,' ').replace(/\s+/g,' ').trim();
const chunks = clean.split(/\s(?:-|\u2013|\u2014)\s/).map(s=>s.trim()).filter(s=>s.length>=5);
return chunks.length ? chunks : [clean];
}
function buildQueries(route) {
if (route === 'A') {
const seeds = [];
const specialist = channelRows.filter(c => c.channel_url && c.name).slice(0, 80);
for (const c of specialist) {
const titles = videoRows.filter(v => v.channel_url === c.channel_url).slice(0, 3).map(v=>v.title);
if (titles.length) seeds.push({query:`${c.name} similar ${config.ecosystem.name} channel`,source_type:'channel',source_ref:c.channel_url,hypothesis:`Find channels in the configured ecosystem similar to ${c.name}; title evidence: ${titles.slice(0,2).join(' / ')}`});
}
return uniqueQueries(seeds, limitOverride || seedsConfigLimit('A'));
}
if (route === 'B') {
const seen = new Set(); const candidates = [];
for (const v of videoRows) {
if (!v.title || v.title.length < 8) continue;
const topic = extractTopics(v.title)[0];
const key = topic.toLowerCase(); if (seen.has(key)) continue; seen.add(key);
candidates.push({query:`${topic} ${config.ecosystem.name} explained`,source_type:'video',source_ref:v.video_id,hypothesis:`High-view observed video title; observed views=${v.view_count ?? 'unknown'}`,priority:Number(v.view_count)||0});
if (candidates.length >= 100) break;
}
candidates.sort((a,b)=>(b.priority||0)-(a.priority||0));
return uniqueQueries(candidates, limitOverride || seedsConfigLimit('B'));
}
if (route === 'C') {
const topics = [...(seeds.topicSeeds || [])];
// Add high-frequency named concepts found in the corpus, while excluding generic noise.
const conceptRows = db.prepare(`SELECT c.canonical_name,COUNT(DISTINCT cc.channel_url) channel_count,COUNT(DISTINCT vc.video_id) video_count,COALESCE(MAX(v.view_count),0) max_views FROM concepts c LEFT JOIN channel_concepts cc ON cc.concept_id=c.id LEFT JOIN video_concepts vc ON vc.concept_id=c.id LEFT JOIN videos v ON v.video_id=vc.video_id WHERE c.ecosystem=? AND c.concept_type IN ('entity','theme','subject','idea','angle') GROUP BY c.id ORDER BY channel_count ASC,video_count DESC,max_views DESC LIMIT 200`).all(config.ecosystem?.name || 'configured-ecosystem');
const manual = topics.slice(0, Math.max(3, Math.floor(seedsConfigLimit('C') / 2))).map(query=>({query,source_type:'manual_topic',hypothesis:'Editable topic seed'}));
const concepts = conceptRows.map(r=>({query:`${r.canonical_name} ${config.ecosystem.name} video`,source_type:'corpus_concept',source_ref:r.canonical_name,hypothesis:`Corpus concept related to ${config.ecosystem.name}; channels=${r.channel_count}, videos=${r.video_count}, maxViews=${r.max_views}`}));
return uniqueQueries([...manual,...concepts,...topics.slice(manual.length).map(query=>({query,source_type:'manual_topic',hypothesis:'Editable topic seed'}))], limitOverride || seedsConfigLimit('C'));
}
return uniqueQueries((seeds.adjacentSeeds || []).map(s=>({query:s.query,source_type:'adjacent_seed',source_ref:s.label,hypothesis:`Adjacent ecosystem: ${s.label}`})), limitOverride || seedsConfigLimit('D'));
}
function seedsConfigLimit(route) { return Number(seeds.routeSettings?.[route]?.maxQueries || 6); }
async function addLLMHypotheses(route, baseQueries) {
if (!useLLM) return baseQueries;
const compact = baseQueries.map((q,i)=>({i,query:q.query,source_type:q.source_type,hypothesis:q.hypothesis}));
const prompt = `Return JSON only: {"queries":[{"query":"...","source_ref":"...","hypothesis":"..."}]}. Generate at most ${Number(seeds.llm?.maxHypothesesPerRoute||4)} additional YouTube search queries for discovery route ${route}. Existing queries and their evidence: ${JSON.stringify(compact.slice(0,12))}. Target ecosystem: "${config.ecosystem.name}". Target description: "${config.ecosystem.description}". Include terms: ${JSON.stringify(config.ecosystem.includeTerms)}. Exclude terms: ${JSON.stringify(config.ecosystem.excludeTerms)}. Adjacent fields: ${JSON.stringify(config.ecosystem.adjacentTerms)}. Goal: find specific channels and high-performing videos relevant to the configured ecosystem, not broad generic results. Avoid duplicating existing queries. Route meanings: A=discover specialist peers of known channels; B=follow unusually successful videos/topics to other creators; C=expand underexplored target topics; D=explore adjacent fields likely to transfer useful methods. Do not invent channel names; queries should be searchable topic phrases.`;
try {
const response = await callLLM({system:'You are a careful research-query planner. Follow the supplied ecosystem configuration, return valid compact JSON only, and prefer precise, discriminative queries over broad ones.',user:prompt,stream:false});
const parsed = parseJsonObject(response.content);
const generated = uniqueQueries((parsed.queries || []).map(q=>({query:q.query,source_type:'llm_hypothesis',source_ref:q.source_ref||null,hypothesis:q.hypothesis||'LLM-generated discovery hypothesis'})), Number(seeds.llm?.maxHypothesesPerRoute||4));
const baseSet = new Set(baseQueries.map(q=>q.query.toLowerCase()));
return uniqueQueries([...baseQueries,...generated.filter(q=>!baseSet.has(q.query.toLowerCase()))], limitOverride || (seedsConfigLimit(route)+Number(seeds.llm?.maxHypothesesPerRoute||4)));
} catch (error) {
console.error(`[discovery-v2][LLM] route=${route} failed; continuing deterministic-only: ${error.message}`);
return baseQueries;
}
}
insertPhase.run(phaseId,now(),JSON.stringify(routes),JSON.stringify({resultsPerQuery,useLLM,delayMs,seedPath}));
console.log('[discovery-v2] phase start',{phaseId,routes,resultsPerQuery,useLLM,knownChannels:channelRows.length,knownVideos:videoRows.length});
let attempted=0,succeeded=0,videoCount=0,channelCount=0,newChannels=0,errorCount=0;
const phaseChannels = new Set();
for (const route of routes) {
let queries = await addLLMHypotheses(route, buildQueries(route));
console.log(`[discovery-v2] route=${route} label=${seeds.routeSettings?.[route]?.label||route} queries=${queries.length}`);
for (const q of queries) {
const query = normalizeQuery(q.query); if (!query) continue;
saveQuery.run(phaseId,route,query,q.source_type||'deterministic',q.source_ref||null,q.hypothesis||null,now());
attempted++;
try {
const results = await searchVideos(query,resultsPerQuery);
const searchQuery = `[discovery-v2:${phaseId}:${route}] ${query}`;
saveSearch.run(searchQuery,now()); const searchId = getSearch.get(searchQuery).id;
let resultPosition=0;
const tx = db.transaction(() => {
for (const video of results) {
resultPosition++;
const channelUrl = videoToChannelUrl(video);
const channelName = video.channel_name || 'Unknown channel';
const wasNew = Boolean(channelUrl && !channelByUrl.has(channelUrl));
if (channelUrl) {
saveChannel.run(channelUrl,channelName,now());
if (wasNew) { channelByUrl.set(channelUrl,{channel_url:channelUrl,name:channelName}); newChannels++; }
phaseChannels.add(channelUrl); channelCount++;
saveEvent.run(phaseId,channelUrl,channelName,route,query,video.video_id,resultPosition,wasNew?1:0,now());
}
const record={video_id:video.video_id,title:video.title||'(untitled)',url:video.url||`https://www.youtube.com/watch?v=${video.video_id}`,channel_url:channelUrl,channel_name:channelName,thumbnail_url:video.thumbnail_url||null,duration:video.duration||null,view_count:parseCount(video.view_count)??(typeof video.view_count==='number'?video.view_count:null),published_at:video.published_at||null,discovered_at:now()};
saveVideo.run(record); saveVideoSearch.run(video.video_id,searchId,resultPosition); videoCount++;
}
});
tx();
updateQuery.run('succeeded',results.length,null,now(),phaseId,route,query); succeeded++;
console.log(`[discovery-v2] route=${route} query=${JSON.stringify(query)} results=${results.length} newChannelsSoFar=${newChannels}`);
} catch (error) {
errorCount++; updateQuery.run('failed',0,String(error?.stack||error).slice(0,2000),now(),phaseId,route,query);
console.error(`[discovery-v2] SEARCH FAILED route=${route} query=${JSON.stringify(query)} error=${error?.message||error}`);
}
if (delayMs) await new Promise(resolve=>setTimeout(resolve,delayMs));
}
}
const finish = now();
db.prepare(`UPDATE discovery_v2_phases SET finished_at=?,status=?,queries_attempted=?,queries_succeeded=?,videos_seen=?,channels_seen=?,channels_new=?,error_count=? WHERE phase_id=?`).run(finish,errorCount?'completed_with_errors':'completed',attempted,succeeded,videoCount,phaseChannels.size,newChannels,errorCount,phaseId);
console.log('[discovery-v2] phase complete',{phaseId,attempted,succeeded,videoCount,uniqueChannels:phaseChannels.size,newChannels,errorCount});
console.log(`[discovery-v2] Generating phase report now...`);
const reportScript = path.resolve(here, '27-discovery-v2-report.js');
const reportRun = spawnSync(process.execPath, [reportScript, `--phase=${phaseId}`], { stdio: 'inherit', cwd: process.cwd() });
if (reportRun.status !== 0) console.error('[discovery-v2] report generation failed; rerun with', `npm run discovery:v2:report -- --phase=${phaseId}`);
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.