Docs/Skills and code/src/
src/stages/20-expand-channel-catalogue.js
import { filterQualifiedRows } from "../analysis/qualification-scope.js";
import { config } from "../../config.js";
import { getDb } from "../db.js";
import { getChannel, getChannelVideos } from "../youtube-channel.js";
const db = getDb();
const arg = prefix => process.argv.find(value => value.startsWith(`${prefix}=`))?.slice(prefix.length + 1);
const numberArg = (name, fallback, min = 0) => {
const raw = arg(`--${name}`);
if (raw == null) return fallback;
const value = Number(raw);
if (!Number.isFinite(value) || value < min) throw new Error(`--${name} must be a number >= ${min}.`);
return Math.floor(value);
};
const limitChannels = numberArg("limit-channels", 25, 1);
const offsetChannels = numberArg("offset-channels", 0, 0);
const limitPerChannel = numberArg("limit-per-channel", 50, 1);
const selectedChannels = db.prepare(`SELECT channel_url, name FROM channels ORDER BY discovered_at, channel_url`).all();
selectedChannels.splice(0, selectedChannels.length, ...filterQualifiedRows(db, selectedChannels, "channel", { label: "20-expand-channel-catalogue.js" }));
selectedChannels.splice(0, selectedChannels.length, ...selectedChannels.slice(offsetChannels, offsetChannels + limitChannels));
const beforeVideoCount = db.prepare(`SELECT COUNT(*) AS n FROM videos`).get().n;
const beforeLinks = db.prepare(`SELECT COUNT(*) AS n FROM channel_videos`).get().n;
const timestamp = () => new Date().toISOString();
const saveVideo = db.prepare(`INSERT INTO videos (
video_id, title, url, channel_url, channel_name, thumbnail_url, duration, view_count, published_at, discovered_at,
published_at_raw, published_at_text, published_at_precision, duration_seconds, duration_precision,
view_count_text, view_count_precision, metadata_observed_at, metadata_source, raw_metadata_json
) VALUES (
@video_id, @title, @url, @channel_url, @channel_name, @thumbnail_url, @duration, @view_count, @published_at, @discovered_at,
@published_at_raw, @published_at_text, @published_at_precision, @duration_seconds, @duration_precision,
@view_count_text, @view_count_precision, @metadata_observed_at, @metadata_source, @raw_metadata_json
) ON CONFLICT(video_id) DO UPDATE SET
title=COALESCE(NULLIF(excluded.title,''), videos.title),
url=COALESCE(excluded.url, videos.url),
channel_url=COALESCE(videos.channel_url, excluded.channel_url),
channel_name=COALESCE(videos.channel_name, excluded.channel_name),
thumbnail_url=COALESCE(videos.thumbnail_url, excluded.thumbnail_url),
duration=COALESCE(videos.duration, excluded.duration),
duration_seconds=COALESCE(videos.duration_seconds, excluded.duration_seconds),
duration_precision=COALESCE(videos.duration_precision, excluded.duration_precision),
view_count=COALESCE(videos.view_count, excluded.view_count),
view_count_text=COALESCE(videos.view_count_text, excluded.view_count_text),
view_count_precision=COALESCE(videos.view_count_precision, excluded.view_count_precision),
published_at=COALESCE(videos.published_at, excluded.published_at),
published_at_raw=COALESCE(videos.published_at_raw, excluded.published_at_raw),
published_at_text=CASE WHEN excluded.published_at_precision LIKE 'exact_%' THEN excluded.published_at_text ELSE COALESCE(videos.published_at_text, excluded.published_at_text) END,
published_at_precision=CASE WHEN excluded.published_at_precision LIKE 'exact_%' THEN excluded.published_at_precision ELSE COALESCE(videos.published_at_precision, excluded.published_at_precision) END,
metadata_observed_at=COALESCE(videos.metadata_observed_at, excluded.metadata_observed_at),
metadata_source=COALESCE(videos.metadata_source, excluded.metadata_source),
raw_metadata_json=COALESCE(videos.raw_metadata_json, excluded.raw_metadata_json)`);
const linkExists = db.prepare(`SELECT 1 FROM channel_videos WHERE channel_url=? AND video_id=? AND source=?`);
const saveLink = db.prepare(`INSERT INTO channel_videos (channel_url, video_id, source, position, discovered_at)
VALUES (@channel_url, @video_id, @source, @position, @discovered_at)
ON CONFLICT(channel_url, video_id, source) DO UPDATE SET position=excluded.position, discovered_at=excluded.discovered_at`);
console.log("[catalogue-expand] Plan", {
selectedChannels: selectedChannels.length, totalChannels: db.prepare(`SELECT COUNT(*) AS n FROM channels`).get().n,
offsetChannels, limitChannels, limitPerChannel, feedMode: "default channel videos feed (source remains unsorted)",
maxChannelPages: config.stage1_5.maxChannelPages,
warning: "This expands the feed sample; it does not claim that Popular/Latest ordering is verified."
});
let channelsFetched = 0, channelsFailed = 0, feedVideos = 0, newVideoIds = 0, newLinks = 0;
const errors = [];
for (let i = 0; i < selectedChannels.length; i++) {
const item = selectedChannels[i];
console.log(`\n[catalogue-expand] Channel ${offsetChannels + i + 1}/${offsetChannels + selectedChannels.length}: ${item.name}`);
console.log(`[catalogue-expand] URL: ${item.channel_url}`);
try {
const channel = await getChannel(item.channel_url);
const fetchedVideos = await getChannelVideos(channel, "", limitPerChannel);
const observedAt = timestamp();
let addedVideoIds = 0;
let addedLinks = 0;
db.transaction(() => {
fetchedVideos.forEach((video, index) => {
const existed = Boolean(db.prepare(`SELECT 1 FROM videos WHERE video_id=?`).get(video.video_id));
saveVideo.run({
...video,
channel_url: item.channel_url,
channel_name: video.channel_name ?? channel.name ?? item.name,
discovered_at: observedAt,
published_at_raw: video.published_at_text ?? video.published_at ?? null,
metadata_observed_at: observedAt,
metadata_source: video.metadata_source ?? "youtubei_channel_feed"
});
const source = fetchedVideos.source_mode ?? "unsorted";
const relationExisted = Boolean(linkExists.get(item.channel_url, video.video_id, source));
saveLink.run({
channel_url: item.channel_url,
video_id: video.video_id,
source,
position: index + 1,
discovered_at: observedAt
});
if (!existed) addedVideoIds++;
if (!relationExisted) addedLinks++;
});
})();
channelsFetched++;
feedVideos += fetchedVideos.length;
newVideoIds += addedVideoIds;
newLinks += addedLinks;
console.log("[catalogue-expand] Saved feed sample", {
channel: item.name, received: fetchedVideos.length, newVideoIds: addedVideoIds,
newOrUpdatedLinks: addedLinks, source: fetchedVideos.source_mode ?? "unsorted",
sortApplied: fetchedVideos.sort_applied ?? false, sortError: fetchedVideos.sort_error ?? null,
first: fetchedVideos[0]?.title ?? null, last: fetchedVideos.at(-1)?.title ?? null
});
} catch (error) {
channelsFailed++;
const message = error instanceof Error ? error.message : String(error);
errors.push({ channel_url: item.channel_url, channel_name: item.name, error: message });
console.error("[catalogue-expand] Channel failed; continuing", { channel: item.name, url: item.channel_url, error: message });
}
if (config.stage1_5.requestDelayMs > 0) await new Promise(resolve => setTimeout(resolve, config.stage1_5.requestDelayMs));
}
const afterVideoCount = db.prepare(`SELECT COUNT(*) AS n FROM videos`).get().n;
const afterLinks = db.prepare(`SELECT COUNT(*) AS n FROM channel_videos`).get().n;
const sourceCounts = db.prepare(`SELECT source, COUNT(*) AS n FROM channel_videos GROUP BY source ORDER BY n DESC`).all();
console.log("\n[catalogue-expand] Summary", {
channelsRequested: selectedChannels.length, channelsFetched, channelsFailed,
feedVideos, newVideoIds, newLinks,
videosBefore: beforeVideoCount, videosAfter: afterVideoCount, netVideoGrowth: afterVideoCount - beforeVideoCount,
linksBefore: beforeLinks, linksAfter: afterLinks, netLinkGrowth: afterLinks - beforeLinks,
relationshipSources: sourceCounts, errors: errors.slice(0, 20),
next: "Run npm run video-metadata -- --limit=500, then npm run normalize:temporal -- --force, embed the new titles, and rebuild priorities/queues."
});
if (channelsFailed) process.exitCode = 1;
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.