`src/stages/27-discovery-v2-report.js`

Code35 lines5.6 KBsrc/
import fs from 'node:fs';
import path from 'node:path';
import { getDb } from '../db.js';
import { ensureDiscoveryV2Schema } from '../discovery/v2-schema.js';
import { parseArgs } from '../discovery/v2-utils.js';

const args=parseArgs(process.argv.slice(2)); const db=ensureDiscoveryV2Schema();
const phases = db.prepare('SELECT * FROM discovery_v2_phases ORDER BY started_at DESC').all();
let phase = args.phase ? phases.find(p => p.phase_id === String(args.phase)) : phases[0];
if (args.phase && !phase && phases.length === 1) {
  phase = phases[0];
  console.warn(`[discovery-v2-report] requested phase "${args.phase}" does not exist; this database has one phase, so using "${phase.phase_id}".`);
}
if (!phase) {
  const available = phases.slice(0, 10).map(p => p.phase_id).join(', ') || '(none)';
  throw new Error(`Discovery phase "${args.phase || ''}" not found. Available phase IDs: ${available}. Omit --phase to use the latest phase.`);
}
const queries=db.prepare('SELECT route,status,COUNT(*) count FROM discovery_v2_queries WHERE phase_id=? GROUP BY route,status ORDER BY route,status').all(phase.phase_id);
const channels=db.prepare(`SELECT e.channel_url,MAX(e.channel_name) channel_name,MIN(e.first_seen_at) first_seen_at,MAX(e.was_new_channel) was_new_channel,COUNT(DISTINCT e.query) query_count,COUNT(DISTINCT e.route) route_count,group_concat(DISTINCT e.route) routes,COUNT(DISTINCT e.first_video_id) observed_videos,MAX(v.view_count) max_observed_views,cm.subscriber_count,cm.video_count,COALESCE((SELECT qlr.classification FROM channel_qualification_llm_results qlr WHERE qlr.channel_url=e.channel_url AND qlr.processing_status='succeeded' ORDER BY qlr.created_at DESC LIMIT 1),cq.classification,'unknown') classification FROM discovery_v2_channel_events e LEFT JOIN videos v ON v.video_id=e.first_video_id LEFT JOIN channels cm ON cm.channel_url=e.channel_url LEFT JOIN channel_qualifications cq ON cq.channel_url=e.channel_url WHERE e.phase_id=? GROUP BY e.channel_url ORDER BY was_new_channel DESC,route_count DESC,query_count DESC,max_observed_views DESC`).all(phase.phase_id);
const errors=db.prepare('SELECT route,query,error FROM discovery_v2_queries WHERE phase_id=? AND status=\'failed\'').all(phase.phase_id);
const outDir=path.resolve('reports/discovery-v2'); fs.mkdirSync(outDir,{recursive:true});
const safe=phase.phase_id.replace(/[^a-zA-Z0-9_-]/g,'_');
const md=[]; md.push(`# Discovery Engine v2 — ${phase.phase_id}`,'',`- Status: ${phase.status}`,`- Routes: ${JSON.parse(phase.routes_json).join(', ')}`,`- Started: ${phase.started_at}`,`- Finished: ${phase.finished_at||'running'}`,`- Queries: ${phase.queries_succeeded}/${phase.queries_attempted} succeeded`,`- Videos observed: ${phase.videos_seen}`,`- Unique channels observed: ${phase.channels_seen}`,`- New channels: ${phase.channels_new}`,`- Search errors: ${phase.error_count}`,'','## Query outcomes','','| Route | Status | Count |','|---|---:|---:|');
for(const q of queries) md.push(`| ${q.route} | ${q.status} | ${q.count} |`);
md.push('','## Newly added channels','','| Channel | Classification | Subscribers | Catalogue videos | Videos seen | Max observed views | Routes | Queries | URL |','|---|---|---:|---:|---:|---:|---|---:|---|');
for(const c of channels.filter(c=>c.was_new_channel)) md.push(`| ${String(c.channel_name||'').replace(/\|/g,'/')} | ${c.classification} | ${c.subscriber_count??'—'} | ${c.video_count??'—'} | ${c.observed_videos} | ${c.max_observed_views??'—'} | ${c.routes||'—'} | ${c.query_count} | ${c.channel_url} |`);
md.push('','## All channels observed in this phase','','| Channel | New? | Classification | Routes | Queries | Videos seen | Max views | URL |','|---|---:|---|---|---:|---:|---:|---|');
for(const c of channels) md.push(`| ${String(c.channel_name||'').replace(/\|/g,'/')} | ${c.was_new_channel?'yes':'no'} | ${c.classification} | ${c.routes||'—'} | ${c.query_count} | ${c.observed_videos} | ${c.max_observed_views??'—'} | ${c.channel_url} |`);
if(errors.length){md.push('','## Failed queries','');for(const e of errors) md.push(`- **${e.route} — ${e.query}**: ${String(e.error||'').replace(/\n/g,' ').slice(0,500)}`);}
md.push('','## Interpretation notes','','- “New” means the channel URL was not present in the `channels` table at the time it was first seen in this phase.','- Search-result counts are not channel catalogue counts. Search rankings are biased and are not evidence of causal performance.','- Scores are computed separately by `discovery:v2:score`; queue rankings should be recomputed after targeted enrichment and qualification.','');
const mdPath=path.join(outDir,`${safe}.md`); fs.writeFileSync(mdPath,md.join('\n'));
const csvPath=path.join(outDir,`${safe}-new-channels.csv`); const cols=['channel_name','channel_url','classification','subscriber_count','video_count','observed_videos','max_observed_views','routes','query_count','was_new_channel'];
const csv=[cols.join(','),...channels.filter(c=>c.was_new_channel).map(c=>cols.map(k=>JSON.stringify(c[k]??'')).join(','))].join('\n'); fs.writeFileSync(csvPath,csv);
console.log('[discovery-v2-report] phase',phase.phase_id); console.log('[discovery-v2-report] summary',{status:phase.status,queries:phase.queries_succeeded+'/'+phase.queries_attempted,videos:phase.videos_seen,channels:phase.channels_seen,newChannels:phase.channels_new,errors:phase.error_count}); console.log('[discovery-v2-report] markdown',mdPath); console.log('[discovery-v2-report] new-channel CSV',csvPath); console.log('[discovery-v2-report] newly added channels:'); for(const c of channels.filter(c=>c.was_new_channel)) console.log(`- ${c.channel_name} | ${c.classification} | ${c.channel_url}`);
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.