`src/stages/13-prioritize-channels.js`

Code160 lines9.4 KBsrc/
import { filterQualifiedRows } from "../analysis/qualification-scope.js";
import crypto from "node:crypto";
import { getDb } from "../db.js";
import { ensureResearchSchema } from "../analysis/research-schema.js";
import { embeddingConfig } from "../analysis/embeddings.js";
import { clusterVideoVectors } from "../analysis/niche-clustering.js";
import { scoreChannels } from "../analysis/channel-priority-scoring.js";

const db = getDb();
ensureResearchSchema();
const embedding = embeddingConfig();
const thresholdArg = process.argv.find(value => value.startsWith("--threshold="));
const threshold = thresholdArg ? Number(thresholdArg.split("=")[1]) : 0.70;
const clusterArg = process.argv.find(value => value.startsWith("--min-cluster-size="));
const parsedMinCluster = clusterArg ? Number(clusterArg.split("=")[1]) : 3;
const videoArg = process.argv.find(value => value.startsWith("--min-videos="));
const parsedMinVideos = videoArg ? Number(videoArg.split("=")[1]) : 5;
if (!Number.isFinite(threshold) || threshold < -1 || threshold > 1) throw new Error("--threshold must be between -1 and 1.");
if (!Number.isFinite(parsedMinCluster) || parsedMinCluster < 2) throw new Error("--min-cluster-size must be at least 2.");
if (!Number.isFinite(parsedMinVideos) || parsedMinVideos < 3) throw new Error("--min-videos must be at least 3.");
const minClusterSize = Math.floor(parsedMinCluster);
const minVideos = Math.floor(parsedMinVideos);
const generatedAt = new Date().toISOString();
const runId = generatedAt.replaceAll(":", "-").replaceAll(".", "-");
const scoringVersion = "channel-priority-v1";
const sha = value => crypto.createHash("sha256").update(String(value ?? "").trim()).digest("hex");
const round = (value, digits = 4) => Number.isFinite(value) ? Number(value.toFixed(digits)) : null;

const embeddingRows = db.prepare(`
  SELECT v.video_id, v.title, ve.vector_json, ve.dimensions, ve.title_hash, ve.input_version
  FROM video_embeddings ve JOIN videos v ON v.video_id=ve.video_id
  WHERE ve.model=?
`).all(embedding.model);
embeddingRows.splice(0, embeddingRows.length, ...filterQualifiedRows(db, embeddingRows, "video", { label: "13-prioritize-channels.js" }));
const currentRows = embeddingRows.filter(row =>
  row.title_hash === sha(row.title) && row.input_version === "title-prefix-v1"
);
const staleCount = embeddingRows.length - currentRows.length;
const dimensionCounts = new Map();
for (const row of currentRows) dimensionCounts.set(row.dimensions, (dimensionCounts.get(row.dimensions) ?? 0) + 1);
if (dimensionCounts.size > 1) {
  throw new Error(`Model ${embedding.model} has inconsistent vector dimensions: ${JSON.stringify(Object.fromEntries(dimensionCounts))}. Re-embed using one model/version.`);
}
const inputVideos = currentRows.map(row => {
  try { return { video_id: row.video_id, title: row.title, vector: JSON.parse(row.vector_json) }; }
  catch { return null; }
}).filter(Boolean);
if (!inputVideos.length) {
  console.error("[priority] No current title embeddings found.", {
    model: embedding.model, storedEmbeddings: embeddingRows.length, staleEmbeddings: staleCount,
    recovery: "Run npm run videos:embed first."
  });
  process.exitCode = 1;
} else {
  console.log("[priority] Starting automatic niche discovery", {
    runId, model: embedding.model, embeddedTitles: inputVideos.length, staleEmbeddings: staleCount,
    dimensions: dimensionCounts.keys().next().value, threshold, minClusterSize, minVideos,
    humanAnnotationsUsedForScoring: false
  });
  const { videos, clusters, membershipByVideo } = clusterVideoVectors(inputVideos, {
    threshold,
    log: (message, detail) => console.log(message, detail)
  });

  const associations = db.prepare(`
    SELECT channel_url, video_id FROM channel_videos
    UNION SELECT channel_url, video_id FROM videos WHERE channel_url IS NOT NULL
  `).all();
  const videoIdsByChannel = new Map();
  const channelUrlsByVideo = new Map();
  for (const row of associations) {
    if (!videoIdsByChannel.has(row.channel_url)) videoIdsByChannel.set(row.channel_url, new Set());
    videoIdsByChannel.get(row.channel_url).add(row.video_id);
    if (!channelUrlsByVideo.has(row.video_id)) channelUrlsByVideo.set(row.video_id, new Set());
    channelUrlsByVideo.get(row.video_id).add(row.channel_url);
  }

  const channelRows = db.prepare(`
    SELECT c.channel_url, c.name,
      COALESCE(m.subscriber_count,c.subscriber_count) AS subscribers,
      COALESCE(m.video_count,c.video_count) AS declared_video_count,
      COALESCE(m.description,c.description) AS description,
      COALESCE(m.keywords_json,c.keywords_json) AS keywords_json
    FROM channels c LEFT JOIN channel_metadata m ON m.channel_url=c.channel_url
    ORDER BY c.name COLLATE NOCASE
  `).all();
  const performanceVideoById = new Map(db.prepare(`
    SELECT video_id, view_count, published_at_normalized, published_at_precision, published_at_raw, discovered_at
    FROM videos
  `).all().map(row => [row.video_id, row]));
  const scores = scoreChannels({
    channelRows, videoIdsByChannel, channelUrlsByVideo, videos, clusters,
    membershipByVideo, performanceVideoById, minVideos, minClusterSize,
    now: new Date(), scoringVersion
  });

  const configuration = {
    threshold, minClusterSize, minVideos, embeddingModel: embedding.model,
    embeddingInputVersion: "title-prefix-v1", scoringVersion,
    weights: { specialist: 0.40, breakthrough: 0.30, novelty: 0.20, exploration: 0.10 },
    algorithm: "deterministic greedy cosine-to-centroid clustering with a seed-similarity guard; run-specific candidate niches, not a definitive taxonomy",
    performance: "breakthrough uses within-channel outlier ratios; views/day is lifetime-average views/day, not current growth velocity",
    humanAnnotationsUsedForScoring: false,
    automationPotentialIncluded: false
  };
  const insertRun = db.prepare(`INSERT INTO channel_priority_runs (run_id, embedding_model, scoring_version, generated_at, configuration_json) VALUES (?, ?, ?, ?, ?)`);
  const insertCluster = db.prepare(`
    INSERT INTO video_niche_clusters (run_id, cluster_no, representative_video_id, representative_title, member_count, channel_count, mean_similarity, centroid_json, diagnostics_json)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
  `);
  const insertMembership = db.prepare(`
    INSERT INTO video_niche_memberships (run_id, video_id, cluster_no, similarity_to_centroid) VALUES (?, ?, ?, ?)
  `);
  const insertScore = db.prepare(`
    INSERT INTO channel_priority_scores (
      run_id, channel_url, specialist_score, breakthrough_score, novelty_score, exploration_score,
      evidence_quality_score, priority_score, confidence, embedded_video_count, sampled_video_count,
      performance_video_count, age_adjusted_performance_count, dominant_cluster_no,
      dominant_cluster_share, mean_title_coherence, rationale_json, generated_at
    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `);
  const timestamp = generatedAt;
  db.transaction(() => {
    insertRun.run(runId, embedding.model, scoringVersion, timestamp, JSON.stringify(configuration));
    for (const cluster of clusters) {
      const stable = cluster.members.length >= minClusterSize;
      insertCluster.run(runId, cluster.clusterNo, cluster.representative.video_id, cluster.representative.title,
        cluster.members.length, cluster.channelCount, round(cluster.meanSimilarity),
        stable ? JSON.stringify([...cluster.centroid]) : "[]",
        JSON.stringify({ stableNicheCandidate: stable, threshold, order: "deterministic-hash-order", seedGuard: true }));
    }
    for (const [videoId, membership] of membershipByVideo) {
      insertMembership.run(runId, videoId, membership.clusterNo, round(membership.similarity, 5));
    }
    for (const score of scores) {
      insertScore.run(runId, score.channelUrl, score.specialistScore, score.breakthroughScore, score.noveltyScore,
        score.explorationScore, score.evidenceQualityScore, score.priorityScore, score.confidence,
        score.embeddedVideoCount, score.sampledVideoCount, score.performanceVideoCount,
        score.ageAdjustedPerformanceCount, score.dominantClusterNo, score.dominantClusterShare,
        score.meanTitleCoherence, JSON.stringify(score.rationale), timestamp);
    }
  })();

  const ranked = scores.filter(score => score.priorityScore != null).sort((a, b) => b.priorityScore - a.priorityScore);
  const specialists = scores.filter(score => score.specialistScore != null).sort((a, b) => b.specialistScore - a.specialistScore).slice(0, 5);
  const breakthroughs = scores.filter(score => score.breakthroughScore != null).sort((a, b) => b.breakthroughScore - a.breakthroughScore).slice(0, 5);
  const novel = scores.filter(score => score.noveltyScore != null).sort((a, b) => b.noveltyScore - a.noveltyScore).slice(0, 5);
  console.log("[priority] Run saved", {
    runId, channelsScored: scores.length, rankableChannels: ranked.length,
    clustersCreated: clusters.length, stableNicheCandidates: clusters.filter(c => c.members.length >= minClusterSize).length,
    embeddedVideos: videos.length,
    topSpecialists: specialists.map(row => ({ channel: row.channel, score: row.specialistScore })),
    topBreakthroughs: breakthroughs.map(row => ({ channel: row.channel, score: row.breakthroughScore, performanceVideos: row.performanceVideoCount })),
    topNovelty: novel.map(row => ({ channel: row.channel, score: row.noveltyScore })),
    topPriority: ranked.slice(0, 10).map(row => ({ channel: row.channel, score: row.priorityScore, confidence: row.confidence })),
    next: "Run npm run priorities:report to write Markdown and JSON reports."
  });
}
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.