Docs/Skills and code/src/

src/analysis/niche-clustering.js

Code92 lines3.7 KBsrc/

import crypto from "node:crypto";

export function unitVector(vector) {
  if (!vector || !vector.length) return null;
  let norm2 = 0;
  for (let i = 0; i < vector.length; i++) norm2 += vector[i] * vector[i];
  const norm = Math.sqrt(norm2);
  if (!norm) return null;
  const result = new Float64Array(vector.length);
  for (let i = 0; i < vector.length; i++) result[i] = vector[i] / norm;
  return result;
}

export function cosineOfUnitVectors(a, b) {
  if (!a || !b || a.length !== b.length) return null;
  let dot = 0;
  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
  return dot;
}

function deterministicOrder(id) {
  return crypto.createHash("sha1").update(id).digest("hex");
}

/** Greedy, run-specific clustering with a seed guard to reduce centroid drift. */
export function clusterVideoVectors(inputVideos, { threshold = 0.70, log = () => {} } = {}) {
  const videos = inputVideos.map(video => ({ ...video, vector: unitVector(video.vector) }))
    .filter(video => video.vector)
    .sort((a, b) => deterministicOrder(a.video_id).localeCompare(deterministicOrder(b.video_id)));
  const clusters = [];
  const membershipByVideo = new Map();
  const logEvery = Math.max(100, Math.floor(videos.length / 10));

  for (let index = 0; index < videos.length; index++) {
    const video = videos[index];
    let bestCluster = null;
    let bestSimilarity = -Infinity;
    for (const cluster of clusters) {
      const centroidSimilarity = cosineOfUnitVectors(video.vector, cluster.centroid);
      const seedSimilarity = cosineOfUnitVectors(video.vector, cluster.seedVector);
      const similarity = centroidSimilarity == null || seedSimilarity == null
        ? null : Math.min(centroidSimilarity, seedSimilarity);
      if (similarity != null && similarity > bestSimilarity) {
        bestSimilarity = similarity;
        bestCluster = cluster;
      }
    }
    if (!bestCluster || bestSimilarity < threshold) {
      const cluster = {
        clusterNo: clusters.length + 1,
        sum: Float64Array.from(video.vector),
        centroid: Float64Array.from(video.vector),
        seedVector: Float64Array.from(video.vector),
        members: [{ video, similarity: 1 }],
        representative: video,
        meanSimilarity: 1,
        channelCount: 0
      };
      clusters.push(cluster);
      membershipByVideo.set(video.video_id, { clusterNo: cluster.clusterNo, similarity: 1 });
    } else {
      for (let i = 0; i < bestCluster.sum.length; i++) bestCluster.sum[i] += video.vector[i];
      bestCluster.centroid = unitVector(bestCluster.sum);
      const similarity = cosineOfUnitVectors(video.vector, bestCluster.centroid) ?? bestSimilarity;
      bestCluster.members.push({ video, similarity });
      membershipByVideo.set(video.video_id, { clusterNo: bestCluster.clusterNo, similarity });
    }
    if ((index + 1) % logEvery === 0 || index + 1 === videos.length) {
      log(`[priority] Cluster assignment ${index + 1}/${videos.length}`, {
        clustersCreated: clusters.length, videosAssigned: index + 1
      });
    }
  }

  for (const cluster of clusters) {
    let similaritySum = 0;
    let bestRepresentativeSimilarity = -Infinity;
    for (const member of cluster.members) {
      const similarity = cosineOfUnitVectors(member.video.vector, cluster.centroid) ?? member.similarity;
      member.similarity = similarity;
      membershipByVideo.set(member.video.video_id, { clusterNo: cluster.clusterNo, similarity });
      similaritySum += similarity;
      if (similarity > bestRepresentativeSimilarity) {
        bestRepresentativeSimilarity = similarity;
        cluster.representative = member.video;
      }
    }
    cluster.meanSimilarity = similaritySum / cluster.members.length;
  }
  return { videos, clusters, membershipByVideo };
}

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.