Docs/Skills and code/src/
src/stages/04-profile-videos.js
import { filterQualifiedRows } from "../analysis/qualification-scope.js";
import { config } from "../../config.js";
import { getDb } from "../db.js";
import { ensureAnalysisSchema } from "../analysis-schema.js";
import { callLLM, parseJsonObject } from "../llm.js";
import { upsertConcept, attachConceptToVideo } from "../concept-store.js";
import { upsertPattern, attachPatternToVideo } from "../pattern-store.js";
const db = getDb();
ensureAnalysisSchema();
const force = process.argv.includes("--force");
const limitArg = process.argv.find(value => value.startsWith("--limit="));
const limit = limitArg ? Number(limitArg.split("=")[1]) : null;
const SOURCE = "video_observation_title_v2";
const videos = db.prepare(`
SELECT
v.video_id,
v.title,
v.channel_url,
v.channel_name,
v.thumbnail_url,
v.view_count,
v.published_at,
q.classification
FROM videos v
LEFT JOIN video_qualifications q ON q.video_id = v.video_id
ORDER BY v.discovered_at
`).all();
videos.splice(0, videos.length, ...filterQualifiedRows(db, videos, "video", { label: "04-profile-videos.js" }));
const saveProfile = db.prepare(`
INSERT INTO video_profiles (
video_id, ecosystem, profile_json, method, model, generated_at
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(video_id) DO UPDATE SET
ecosystem = excluded.ecosystem,
profile_json = excluded.profile_json,
method = excluded.method,
model = excluded.model,
generated_at = excluded.generated_at
`);
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function cleanProposal(value) {
return asArray(value)
.map(item => {
if (typeof item === "string") {
return { name: item.trim(), evidence: [], confidence: null };
}
if (!item || typeof item !== "object") return null;
const name = String(item.name ?? "").trim();
if (!name) return null;
return {
name,
evidence: [...new Set(asArray(item.evidence)
.map(x => String(x).trim())
.filter(Boolean))],
confidence: Number.isFinite(Number(item.confidence))
? Math.max(0, Math.min(1, Number(item.confidence)))
: null
};
})
.filter(Boolean);
}
function buildPrompt(video) {
return `Analyze this YouTube video as TITLE-LEVEL research data.
This stage does NOT watch the video. Treat the title as the primary evidence.
The goal is to produce reusable vocabulary observations, not a final ontology.
Return ONLY valid JSON:
{
"schema_version": "video_observation_title_v2",
"evidence_level": "title_only",
"vocabulary_proposals": {
"entities": [
{"name": "...", "entity_type": "person|organization|place|product|concept|event|work|other", "evidence": ["title"], "confidence": 0.0}
],
"themes": [
{"name": "...", "evidence": ["title"], "confidence": 0.0}
],
"angles": [
{"name": "...", "evidence": ["title"], "confidence": 0.0}
],
"formats": [
{"name": "...", "evidence": ["title"], "confidence": 0.0}
],
"hooks": [
{"name": "...", "evidence": ["title"], "confidence": 0.0}
],
"promises": [
{"name": "...", "evidence": ["title"], "confidence": 0.0}
],
"packaging": [
{"name": "...", "evidence": ["title"], "confidence": 0.0}
]
},
"limitations": ["what cannot be known from the title"],
"confidence": 0.0
}
Rules:
- Entities are identifiable things explicitly named or strongly identifiable from the title.
- Themes must be supported by the title; do not turn generic nouns into themes.
- Angles describe the editorial framing explicitly suggested by the title.
- Formats describe the apparent content form only when the title supports it.
- Hooks describe title-level attention mechanisms, not performance claims.
- Promises describe what a viewer is being promised by the title.
- Packaging describes reusable title structures, not exact wording.
- Do not claim to know the video's arguments, evidence, narrative, or actual contents.
- Do not infer thumbnail style; thumbnail analysis is a separate stage.
- Evidence must be "title" for every proposal.
- Keep each list to at most 8 items.
- Preserve distinctions. Canonicalization is a later stage.
- Confidence is confidence in the extraction, not truth probability.
VIDEO
Title: ${video.title}
Channel: ${video.channel_name ?? ""}
Published: ${video.published_at ?? ""}
Views: ${video.view_count ?? ""}
Classification: ${video.classification ?? "unknown"}`;
}
function saveExtracted(video, observation, model) {
const now = new Date().toISOString();
const vocabulary = observation.vocabulary_proposals ?? {};
db.transaction(() => {
if (force) {
db.prepare(`
DELETE FROM video_concepts
WHERE video_id = ? AND source IN ('video_profile_llm', ?)
`).run(video.video_id, SOURCE);
db.prepare(`
DELETE FROM video_patterns
WHERE video_id = ? AND source IN ('video_profile_llm', ?)
`).run(video.video_id, SOURCE);
}
saveProfile.run(
video.video_id,
config.ecosystem.name,
JSON.stringify(observation),
"llm_observation",
model,
now
);
for (const item of cleanProposal(vocabulary.entities)) {
const concept = upsertConcept({
ecosystem: config.ecosystem.name,
name: item.name,
type: "entity",
source: SOURCE
});
if (concept) {
attachConceptToVideo({
videoId: video.video_id,
conceptId: concept.id,
source: SOURCE,
confidence: item.confidence
});
}
}
for (const item of cleanProposal(vocabulary.themes)) {
const concept = upsertConcept({
ecosystem: config.ecosystem.name,
name: item.name,
type: "theme",
source: SOURCE
});
if (concept) {
attachConceptToVideo({
videoId: video.video_id,
conceptId: concept.id,
source: SOURCE,
confidence: item.confidence
});
}
}
for (const item of cleanProposal(vocabulary.angles)) {
const concept = upsertConcept({
ecosystem: config.ecosystem.name,
name: item.name,
type: "angle",
source: SOURCE
});
if (concept) {
attachConceptToVideo({
videoId: video.video_id,
conceptId: concept.id,
source: SOURCE,
confidence: item.confidence
});
}
}
for (const [type, key] of [
["format", "formats"],
["hook", "hooks"],
["promise", "promises"],
["packaging", "packaging"]
]) {
for (const item of cleanProposal(vocabulary[key])) {
const pattern = upsertPattern({
ecosystem: config.ecosystem.name,
type,
name: item.name
});
if (pattern) {
attachPatternToVideo({
videoId: video.video_id,
patternId: pattern.id,
source: SOURCE,
confidence: item.confidence
});
}
}
}
})();
}
let processed = 0;
let skipped = 0;
let failed = 0;
for (const [index, video] of videos.entries()) {
if (limit !== null && processed >= limit) break;
const existing = db.prepare(`
SELECT 1 FROM video_profiles
WHERE video_id = ? AND ecosystem = ?
`).get(video.video_id, config.ecosystem.name);
if (existing && !force) {
skipped++;
continue;
}
console.log(`\n=== Video observation ${index + 1}/${videos.length} ===`);
console.log(`Video: ${video.title}`);
console.log(`ID: ${video.video_id}`);
console.log(`Evidence level: title_only`);
try {
const llm = await callLLM({
system: "You are a careful research data extractor. Return only the requested JSON. Do not hallucinate, overclaim, or silently canonicalize.",
user: buildPrompt(video),
temperature: 0.1,
maxTokens: 1800
});
const observation = parseJsonObject(llm.content);
if (observation.schema_version !== "video_observation_title_v2") {
throw new Error(`Unexpected schema_version: ${observation.schema_version ?? "missing"}`);
}
saveExtracted(video, observation, llm.model);
const v = observation.vocabulary_proposals ?? {};
console.log("Observation saved:", {
entities: cleanProposal(v.entities).length,
themes: cleanProposal(v.themes).length,
angles: cleanProposal(v.angles).length,
formats: cleanProposal(v.formats).length,
hooks: cleanProposal(v.hooks).length,
promises: cleanProposal(v.promises).length,
packaging: cleanProposal(v.packaging).length
});
processed++;
} catch (error) {
failed++;
console.error("Video observation failed:", error instanceof Error ? error.message : error);
}
}
console.log("\n=== Video observation summary ===");
console.log({ processed, skipped, failed });
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.