Docs/Skills and code/src/
src/stages/09-build-vocabulary.js
import { filterQualifiedRows } from "../analysis/qualification-scope.js";
import { config } from "../../config.js";
import { getDb } from "../db.js";
import { ensureResearchSchema } from "../analysis/research-schema.js";
import { addVocabularyEvidence, cleanLabel, upsertVocabularyTerm } from "../analysis/vocabulary-store.js";
import { cosineSimilarity, embedTexts, embeddingConfig } from "../analysis/embeddings.js";
const db = getDb();
ensureResearchSchema();
const ecosystem = config.ecosystem.name;
const noEmbeddings = process.argv.includes("--no-embeddings");
const includeExpressions = process.argv.includes("--include-expressions");
const limitArg = process.argv.find(value => value.startsWith("--limit="));
const limit = limitArg ? Math.max(1, Number(limitArg.split("=")[1])) : 250;
const thresholdArg = process.argv.find(value => value.startsWith("--min-similarity="));
const minSimilarity = thresholdArg ? Number(thresholdArg.split("=")[1]) : 0.78;
const embedding = embeddingConfig();
const now = () => new Date().toISOString();
let candidateEvidenceAdded = 0;
let rejectedNoise = 0;
function parseJson(value, fallback = {}) {
try { return JSON.parse(value); } catch { return fallback; }
}
function isUsefulLabel(type, label) {
if (!label || label.length < 2 || label.length > 140) return false;
if (/^(unknown|none|null|n\/a|not sure|other)$/i.test(label)) return false;
if (type === "entity") {
if (/^\$?\s*\d[\d,.]*(?:\s*[kmb])?(?:\s*(?:trillion|billion|million))?$/i.test(label)) return false;
if (/^\d+\s*(?:s|sec|seconds?|mins?|minutes?|hours?|days?)$/i.test(label)) return false;
if (/^(rich people|wealthy people|people|everyone|history|science|education|official channel|documentary|explained)$/i.test(label)) return false;
}
if (type === "format_family" && (/^\d+\s*(?:minutes?|mins?|hours?|seconds?)$/i.test(label) || /(?:explained in|in)\s*\d+\s*(?:minutes?|mins?)/i.test(label))) return false;
return true;
}
function entriesFromObservation(observation) {
const result = [];
const push = (type, value, meta = {}) => {
const label = cleanLabel(value);
if (!label) return;
if (!isUsefulLabel(type, label)) { rejectedNoise++; return; }
result.push({ type, label, confidence: meta.confidence ?? observation.confidence ?? null, meta });
};
const list = (type, values) => {
if (!Array.isArray(values)) return;
for (const value of values) {
if (value && typeof value === "object") {
const itemType = type === "entity" ? String(value.entity_type ?? value.type ?? "entity") : type;
push(type === "entity" ? "entity" : type, value, { ...value, entityType: itemType });
} else push(type, value);
}
};
list("entity", observation.entities);
list("theme", observation.themes);
list("angle", observation.angles);
if (observation.angle) push("angle", observation.angle);
if (observation.format_family) push("format_family", observation.format_family);
else if (observation.format) push("format_family", observation.format);
if (observation.hook_mechanism) push("hook_mechanism", observation.hook_mechanism);
else if (includeExpressions && observation.hook) push("hook_text", observation.hook);
if (includeExpressions && observation.hook_text) push("hook_text", observation.hook_text);
if (observation.promise_type) push("promise_type", observation.promise_type);
if (includeExpressions && observation.promise_text) push("promise_text", observation.promise_text);
else if (includeExpressions && observation.promise) push("promise_text", observation.promise);
if (observation.packaging_pattern) push("packaging_pattern", observation.packaging_pattern);
else if (observation.packaging) push("packaging_pattern", observation.packaging);
list("theme", observation.concepts);
return result;
}
function saveEvidence({ type, label, objectType, objectId, source, evidence, confidence }) {
const term = upsertVocabularyTerm({ ecosystem, type, label });
if (!term) return;
const inserted = addVocabularyEvidence({
termId: term.term_id, objectType, objectId, source, rawLabel: label,
evidence: { ...evidence, confidence: confidence == null || confidence === "" ? null : (Number.isFinite(Number(confidence)) ? Number(confidence) : null) }
});
candidateEvidenceAdded += inserted;
}
function ingestVideoObservations() {
const rows = db.prepare(`
SELECT video_id, channel_url, source, observation_json, confidence
FROM video_observations WHERE ecosystem=? ORDER BY created_at
`).all(ecosystem);
rows.splice(0, rows.length, ...filterQualifiedRows(db, rows, "video", { label: "09-build-vocabulary.js" }));
for (const row of rows) {
const observation = parseJson(row.observation_json);
for (const entry of entriesFromObservation(observation)) {
saveEvidence({ ...entry, objectType: "video", objectId: row.video_id, source: row.source,
evidence: { evidence_level: observation.evidence_level ?? "title_only", title: observation.title ?? null, field: entry.type, entityType: entry.meta?.entityType ?? null } });
}
}
console.log(`[vocabulary] Read ${rows.length} video observations.`);
}
function ingestChannelProfiles() {
const rows = db.prepare(`
SELECT channel_url, profile_json, generated_at FROM channel_profiles WHERE ecosystem=?
`).all(ecosystem);
rows.splice(0, rows.length, ...filterQualifiedRows(db, rows, "channel", { label: "09-build-vocabulary.js" }));
for (const row of rows) {
const profile = parseJson(row.profile_json);
const vocab = profile.vocabulary_proposals ?? {};
const groups = [
["entity", vocab.entities], ["theme", vocab.themes], ["angle", vocab.angles],
["format_family", vocab.formats], ["hook_mechanism", vocab.hooks],
["promise_type", vocab.promises], ["packaging_pattern", vocab.packaging]
];
for (const [type, values] of groups) {
for (const value of Array.isArray(values) ? values : []) {
const label = cleanLabel(value);
if (!isUsefulLabel(type, label)) { if (label) rejectedNoise++; continue; }
saveEvidence({ type, label, objectType: "channel", objectId: row.channel_url,
source: "channel_profile:" + (profile.schema_version ?? "unknown"),
confidence: value?.confidence ?? profile.confidence,
evidence: { evidence: value?.evidence ?? [], entityType: value?.entity_type ?? null, generatedAt: row.generated_at } });
}
}
}
console.log(`[vocabulary] Read ${rows.length} channel profiles.`);
}
function ingestLegacyVocabulary() {
const channelConcepts = db.prepare(`
SELECT c.concept_type AS type, c.canonical_name AS label, cc.channel_url AS object_id,
cc.source, cc.confidence FROM channel_concepts cc JOIN concepts c ON c.id=cc.concept_id
WHERE c.ecosystem=?
`).all(ecosystem);
channelConcepts.splice(0, channelConcepts.length, ...filterQualifiedRows(db, channelConcepts, "channel", { label: "09-build-vocabulary.js" }));
const videoConcepts = db.prepare(`
SELECT c.concept_type AS type, c.canonical_name AS label, vc.video_id AS object_id,
vc.source, vc.confidence FROM video_concepts vc JOIN concepts c ON c.id=vc.concept_id
WHERE c.ecosystem=?
`).all(ecosystem);
const channelPatterns = db.prepare(`
SELECT p.pattern_type AS type, p.canonical_name AS label, cp.channel_url AS object_id,
cp.source, cp.confidence FROM channel_patterns cp JOIN patterns p ON p.id=cp.pattern_id
WHERE p.ecosystem=?
`).all(ecosystem);
const videoPatterns = db.prepare(`
SELECT p.pattern_type AS type, p.canonical_name AS label, vp.video_id AS object_id,
vp.source, vp.confidence FROM video_patterns vp JOIN patterns p ON p.id=vp.pattern_id
WHERE p.ecosystem=?
`).all(ecosystem);
for (const row of [...channelConcepts, ...videoConcepts, ...channelPatterns, ...videoPatterns]) {
const type = ({ subject: "entity", idea: "angle", format: "format_family", hook: "hook_mechanism", promise: "promise_type", packaging: "packaging_pattern" })[row.type] ?? row.type;
const objectType = row.object_id?.startsWith?.("https://") ? "channel" : "video";
if (!isUsefulLabel(type, row.label)) { rejectedNoise++; continue; }
saveEvidence({ type, label: row.label, objectType, objectId: row.object_id,
source: `legacy:${row.source}`, confidence: row.confidence,
evidence: { sourceTable: type === row.type ? "legacy_vocabulary" : "legacy_vocabulary_type_mapped" } });
}
console.log(`[vocabulary] Read legacy vocabulary links: ${channelConcepts.length + videoConcepts.length + channelPatterns.length + videoPatterns.length}.`);
}
async function embedPendingTerms() {
const pending = db.prepare(`
SELECT t.term_id, t.term_type, t.label,
COUNT(DISTINCT e.evidence_id) AS evidence_count,
MAX(CASE WHEN a.important=1 THEN 1 ELSE 0 END) AS important_evidence
FROM vocabulary_terms t
LEFT JOIN vocabulary_evidence e ON e.term_id=t.term_id
LEFT JOIN editorial_annotations a ON a.object_type='channel' AND a.object_id=e.object_id AND e.object_type='channel'
LEFT JOIN vocabulary_embeddings ve ON ve.term_id=t.term_id AND ve.model=?
WHERE t.ecosystem=? AND ve.term_id IS NULL AND t.status <> 'rejected'
GROUP BY t.term_id
ORDER BY important_evidence DESC, evidence_count DESC, t.term_id
LIMIT ?
`).all(embedding.model, ecosystem, limit);
if (!pending.length) {
console.log(`[vocabulary] No pending terms need embeddings for model ${embedding.model}.`);
return [];
}
console.log(`[vocabulary] Embedding ${pending.length} typed labels with ${embedding.model} at ${embedding.baseUrl}.`);
const vectorsById = new Map();
for (let start = 0; start < pending.length; start += 32) {
const batch = pending.slice(start, start + 32);
console.log(`[vocabulary] Embedding batch ${Math.floor(start / 32) + 1}/${Math.ceil(pending.length / 32)} (${batch.length} labels).`);
const vectors = await embedTexts(batch.map(term => `${term.term_type}: ${term.label}`), embedding);
batch.forEach((term, index) => vectorsById.set(term.term_id, vectors[index]));
}
const insert = db.prepare(`
INSERT INTO vocabulary_embeddings (term_id, model, dimensions, vector_json, created_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(term_id, model) DO UPDATE SET
dimensions=excluded.dimensions, vector_json=excluded.vector_json, created_at=excluded.created_at
`);
const tx = db.transaction(() => {
for (const term of pending) {
const vector = vectorsById.get(term.term_id);
insert.run(term.term_id, embedding.model, vector.length, JSON.stringify(vector), now());
}
});
tx();
return pending.map(term => ({ ...term, vector: vectorsById.get(term.term_id) }));
}
function proposeSimilarities(newTerms) {
if (!newTerms.length) return 0;
const all = db.prepare(`
SELECT t.term_id, t.term_type, t.label, e.vector_json
FROM vocabulary_terms t JOIN vocabulary_embeddings e ON e.term_id=t.term_id
WHERE t.ecosystem=? AND e.model=? AND t.status <> 'rejected'
`).all(ecosystem, embedding.model).map(row => ({ ...row, vector: parseJson(row.vector_json, []) }));
const allByType = new Map();
for (const item of all) {
if (!allByType.has(item.term_type)) allByType.set(item.term_type, []);
allByType.get(item.term_type).push(item);
}
const insert = db.prepare(`
INSERT OR IGNORE INTO vocabulary_relations (
term_a_id, term_b_id, relation_type, similarity, status, method, evidence_json, created_at, updated_at
) VALUES (?, ?, ?, ?, 'proposed', ?, ?, ?, ?)
`);
let proposals = 0;
const tx = db.transaction(() => {
for (const fresh of newTerms) {
const candidates = (allByType.get(fresh.term_type) ?? [])
.filter(item => item.term_id !== fresh.term_id)
.map(item => ({ item, score: cosineSimilarity(fresh.vector, item.vector) }))
.filter(pair => pair.score != null && pair.score >= minSimilarity)
.sort((a, b) => b.score - a.score).slice(0, 8);
for (const { item, score } of candidates) {
const a = Math.min(fresh.term_id, item.term_id);
const b = Math.max(fresh.term_id, item.term_id);
const relationType = score >= 0.9 ? "possible_alias" : "related_to";
const timestamp = now();
const result = insert.run(a, b, relationType, score, `cosine:${embedding.model}:v1`,
JSON.stringify({ freshTermId: fresh.term_id, sameTypeOnly: true, automaticMerge: false }), timestamp, timestamp);
proposals += result.changes;
}
}
});
tx();
return proposals;
}
console.log("[vocabulary] Starting additive vocabulary build", { ecosystem, limit, noEmbeddings, includeExpressions, model: embedding.model, minSimilarity });
ingestVideoObservations();
ingestChannelProfiles();
ingestLegacyVocabulary();
console.log("[vocabulary] Candidate ingestion complete", {
evidenceRowsAdded: candidateEvidenceAdded, rejectedNoisyLabels: rejectedNoise,
terms: db.prepare("SELECT COUNT(*) AS n FROM vocabulary_terms WHERE ecosystem=?").get(ecosystem).n
});
if (noEmbeddings) {
console.log("[vocabulary] Embedding skipped by --no-embeddings. Rerun without this flag when ready.");
} else {
try {
const newlyEmbedded = await embedPendingTerms();
const proposed = proposeSimilarities(newlyEmbedded);
console.log("[vocabulary] Embedding/similarity phase complete", {
embeddedThisRun: newlyEmbedded.length, model: embedding.model, similarityProposalsAdded: proposed,
note: "Proposals are not merges; only same-type terms are compared."
});
} catch (error) {
console.error("[vocabulary] Candidate terms were saved, but embedding failed.", {
model: embedding.model, baseUrl: embedding.baseUrl,
error: error instanceof Error ? error.message : String(error),
recovery: "Start Ollama and pull the configured embedding model, then rerun this stage. Existing candidates are retained."
});
process.exitCode = 1;
}
}
Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.