Docs/Skills and code/src/

src/llm.js

Code390 lines10.9 KBsrc/

Mentioned on: page 01, page 12

import { TextDecoder } from "util";
import {
  GeminiError, callGeminiWithRetry, explainGeminiError, getGeminiConfig, secondsUntilPacificMidnight
} from "./gemini.js";

// ============================================================
// LLM ADAPTER
//
// callLLM() sends to Gemma 4 on the Google Gemini API by default and falls back to the
// Ollama model when Gemini cannot answer (daily limit, a rate limit that does not clear,
// overload, bad key, timeout). Ollama is used directly when there is no Gemini key.
//
//   LLM_PROVIDER   gemini | ollama   default: gemini when GEMINI_API_KEY is set, else ollama
//   LLM_FALLBACK   off               disable the Ollama fallback (default: on)
//   GEMINI_MODEL (gemma-4-26b-a4b-it; gemma-4-31b-it is the larger but slower, flakier option),
//   GEMINI_THINKING (minimal default; high, or "default" for the model's own setting),
//   GEMINI_MAX_OUTPUT_TOKENS (16000), GEMINI_TEMPERATURE (0.1), GEMINI_TIMEOUT_S (300),
//   GEMINI_MAX_RETRIES (2), GEMINI_MAX_WAIT_S (90), GEMINI_JSON_MODE=off (JSON mode is on by default)
//   OLLAMA_HOST, OLLAMA_MODEL
// Variables come from the environment or the project .env file (see .env.example).
// ============================================================

function camelToSnake(key) {
  return key.replace(/([A-Z])/g, "_$1").toLowerCase();
}

export function normalizeKeys(value) {
  if (Array.isArray(value)) {
    return value.map(normalizeKeys);
  }

  if (value && typeof value === "object") {
    const normalized = {};
    for (const key of Object.keys(value)) {
      const snakeKey = camelToSnake(key);
      normalized[snakeKey] = normalizeKeys(value[key]);
    }
    return normalized;
  }

  return value;
}

function extractJsonCandidate(text) {
  if (!text || typeof text !== "string") {
    return null;
  }

  const trimmed = text.trim();
  const fenceMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
  if (fenceMatch) {
    // Models sometimes leave stray characters after the object inside the fence
    // (e.g. a lone quote before the closing ```); keep only the balanced object.
    const inner = fenceMatch[1].trim();
    return scanBalancedJson(inner) ?? inner;
  }

  return scanBalancedJson(trimmed) ?? trimmed;
}

function scanBalancedJson(trimmed) {
  let started = false;
  let depth = 0;
  let start = -1;
  let openChar = "";
  let closeChar = "";
  let inString = false;
  let escaped = false;

  for (let i = 0; i < trimmed.length; i++) {
    const ch = trimmed[i];

    if (started) {
      if (inString) {
        if (escaped) {
          escaped = false;
        } else if (ch === "\\") {
          escaped = true;
        } else if (ch === '"') {
          inString = false;
        }
        continue;
      }

      if (ch === '"') {
        inString = true;
        continue;
      }

      if (ch === openChar) {
        depth += 1;
      } else if (ch === closeChar) {
        depth -= 1;
        if (depth === 0) {
          return trimmed.slice(start, i + 1).trim();
        }
      }

      continue;
    }

    if (ch === "{" || ch === "[") {
      started = true;
      depth = 1;
      start = i;
      openChar = ch;
      closeChar = ch === "{" ? "}" : "]";
    }
  }

  return null;
}

function cleanJsonString(rawText) {
  if (typeof rawText !== "string") {
    return rawText;
  }

  let candidate = extractJsonCandidate(rawText);
  if (!candidate) {
    return rawText;
  }

  candidate = candidate.trim();
  candidate = candidate.replace(/```(?:json)?/gi, "").replace(/```/g, "").trim();
  candidate = candidate.replace(/,\s*([}\]])/g, "$1");

  return candidate;
}

// Gemma (Gemini API, minimal thinking) occasionally mangles the colon after a key:
// `"confidence：0.95` (full-width colon) or `"confidence:0.95` (closing quote dropped).
// Only used after a normal parse has failed, and the result still has to parse.
function repairKeySeparators(text) {
  return text
    .replace(/"([A-Za-z_][A-Za-z0-9_]*)[:：]\s*(?=[-0-9"\[{]|true\b|false\b|null\b)/g, '"$1":')
    .replace(/([,{]\s*)([A-Za-z_][A-Za-z0-9_]*)":/g, '$1"$2":');
}

export function parseJson(rawText) {
  const cleaned = cleanJsonString(rawText);
  const attempts = [
    cleaned,
    cleaned.replace(/[\r\n]+/g, " "),
    repairKeySeparators(cleaned).replace(/[\r\n]+/g, " ")
  ];

  for (const candidate of attempts) {
    try {
      return normalizeKeys(JSON.parse(candidate));
    } catch {
      // Try the next, more forgiving form.
    }
  }

  throw new Error(
    `Unable to parse JSON from LLM output. Raw output:\n${rawText}`
  );
}

export function getOllamaConfig() {
  return {
    host: process.env.OLLAMA_HOST || "http://localhost:11434",
    model: process.env.OLLAMA_MODEL || "gemma4:31b-cloud"
  };
}

export async function callOllama(prompt, { stream = true } = {}) {
  const { host, model } = getOllamaConfig();

  const res = await fetch(`${host.replace(/\/$/, "")}/api/generate`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model,
      prompt,
      stream
    })
  });

  if (!res.ok) {
    const text = await res.text();
    throw new Error(
      `Ollama request failed: ${res.status} ${res.statusText} - ${text}`
    );
  }

  if (!stream) {
    const data = await res.json();
    return data.response;
  }

  if (!res.body) {
    throw new Error("Ollama response has no body to stream");
  }

  const reader = res.body.getReader();
  const decoder = new TextDecoder();
  let buffer = "";
  let result = "";

  process.stdout.write("[LLM] ");

  const flushLine = (line) => {
    const trimmed = line.trim();
    if (!trimmed || trimmed === "[DONE]") {
      return;
    }

    let payloadText = trimmed;
    if (trimmed.startsWith("data:")) {
      payloadText = trimmed.slice(5).trim();
    }

    try {
      const payload = JSON.parse(payloadText);
      const text =
        payload.response ||
        payload.output ||
        payload.text ||
        payload.data ||
        "";

      if (text) {
        process.stdout.write(text);
        result += text;
      }
    } catch {
      process.stdout.write(payloadText);
      result += payloadText;
    }
  };

  while (true) {
    const { done, value } = await reader.read();
    if (done) break;

    buffer += decoder.decode(value, { stream: true });
    const lines = buffer.split(/\r?\n/);
    buffer = lines.pop() || "";

    for (const line of lines) {
      flushLine(line);
    }
  }

  buffer += decoder.decode();
  if (buffer) {
    flushLine(buffer);
  }

  process.stdout.write("\n");
  return result.trim();
}

export function getLLMProvider() {
  const requested = (process.env.LLM_PROVIDER || "").trim().toLowerCase();

  if (requested === "ollama" || requested === "gemini") {
    return requested;
  }

  return getGeminiConfig().apiKey ? "gemini" : "ollama";
}

const fallbackEnabled = () => !/^(off|false|0|no)$/i.test((process.env.LLM_FALLBACK || "").trim());
const numberFromEnv = (name, fallback) => {
  const value = Number(process.env[name]);
  return process.env[name] && Number.isFinite(value) ? value : fallback;
};

// Which backend callLLM() tries first, and what it falls back to.
export function getLLMConfig() {
  const provider = getLLMProvider();
  const ollama = getOllamaConfig();

  return {
    provider,
    model: provider === "gemini" ? getGeminiConfig().model : ollama.model,
    host: ollama.host,
    fallback: provider === "gemini" && fallbackEnabled() ? { provider: "ollama", model: ollama.model } : null
  };
}

// After a failure that will keep failing, skip Gemini for a while instead of paying
// its retry waits on every batch. Per process only.
let geminiPausedUntil = 0;
let geminiPauseReason = "";

export function resetLLMState() {
  geminiPausedUntil = 0;
  geminiPauseReason = "";
}

function pauseGemini(error) {
  const seconds = {
    "daily-limit": secondsUntilPacificMidnight(),
    auth: Infinity,
    "no-key": Infinity,
    "requests-per-minute-limit": 300,
    "tokens-per-minute-limit": 300,
    "rate-limit-unknown": 300,
    overloaded: 60,
    "server-error": 60
  }[error?.kind];

  if (seconds) {
    geminiPausedUntil = seconds === Infinity ? Infinity : Date.now() + seconds * 1000;
    geminiPauseReason = error.kind;
  }
}

// Thinking tokens made up over half of the generated tokens and doubled the latency, with no
// quality loss seen in the 16-channel comparison, so thinking is minimal unless overridden.
function thinkingLevelFromEnv() {
  const value = (process.env.GEMINI_THINKING || "minimal").trim().toLowerCase();
  return ["default", "off", "none", ""].includes(value) ? "" : value;
}

async function callGeminiBackend(prompt, { stream }) {
  const { model } = getGeminiConfig();

  const result = await callGeminiWithRetry({
    // One combined prompt string, exactly like the Ollama path.
    user: prompt,
    model,
    stream,
    maxOutputTokens: numberFromEnv("GEMINI_MAX_OUTPUT_TOKENS", 16000),
    temperature: numberFromEnv("GEMINI_TEMPERATURE", 0.1),
    thinkingLevel: thinkingLevelFromEnv(),
    jsonMode: !/^(off|false|0|no)$/i.test((process.env.GEMINI_JSON_MODE || "").trim()),
    timeoutMs: numberFromEnv("GEMINI_TIMEOUT_S", 300) * 1000
  }, {
    maxRetries: numberFromEnv("GEMINI_MAX_RETRIES", 2),
    maxWaitSeconds: numberFromEnv("GEMINI_MAX_WAIT_S", 90),
    onWait: ({ error, waitSeconds }) =>
      console.warn(`[LLM] Gemini ${error.kind}; waiting ${waitSeconds}s before retrying`)
  });

  return { content: result.content, model, provider: "gemini", usage: result.usage, fellBack: false };
}

export async function callLLM({
  system = "",
  user = "",
  stream = true
} = {}) {
  const prompt = [
    system?.trim(),
    user?.trim()
  ].filter(Boolean).join("\n\n");

  let fellBackBecause = null;

  if (getLLMProvider() === "gemini") {
    if (Date.now() < geminiPausedUntil) {
      fellBackBecause = `paused after ${geminiPauseReason}`;
    } else {
      try {
        return await callGeminiBackend(prompt, { stream });
      } catch (error) {
        if (!fallbackEnabled()) {
          throw error;
        }

        pauseGemini(error);
        fellBackBecause = explainGeminiError(error).split("\n")[0].slice(0, 300);
      }
    }

    if (!fallbackEnabled()) {
      throw new GeminiError(`Gemini is ${fellBackBecause} and LLM_FALLBACK is off`, { kind: "paused" });
    }

    console.warn(`[LLM] Gemini unavailable (${fellBackBecause}); using Ollama ${getOllamaConfig().model}`);
  }

  const { model } = getOllamaConfig();
  const content = await callOllama(prompt, { stream });

  return {
    content,
    model,
    provider: "ollama",
    fellBack: fellBackBecause !== null
  };
}

export const parseJsonObject = parseJson;

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.