@plur-ai/core - npm Package Compare versions

Comparing version 0.17.1 to 0.17.2

dist/chunk-SYCJM6JJ.js

+// src/fts.ts
+import { createHash } from "crypto";
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "the",
+  "and",
+  "for",
+  "that",
+  "this",
+  "with",
+  "from",
+  "are",
+  "was",
+  "were",
+  "been",
+  "have",
+  "has",
+  "not",
+  "but",
+  "its",
+  "you",
+  "your",
+  "can",
+  "will",
+  "should",
+  "would",
+  "could",
+  "may",
+  "might"
+]);
+var MIN_TOKEN_LENGTH = 2;
+var TOKENIZER_VERSION = 2;
+function ftsTokenize(text) {
+  const lower = text.toLowerCase();
+  const tokens = lower.replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
+  for (const run of lower.match(new RegExp("\\p{Script=Han}{2,}", "gu")) ?? []) {
+    for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
+  }
+  return tokens;
+}
+function engramSearchText(engram) {
+  const parts = [engram.statement];
+  if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
+  if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
+  if (engram.entities) {
+    for (const e of engram.entities) {
+      parts.push(e.name);
+      if (e.type !== "other") parts.push(e.type);
+    }
+  }
+  if (engram.temporal) {
+    if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
+    if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
+  }
+  if (engram.rationale) parts.push(engram.rationale);
+  if (engram.source) parts.push(engram.source);
+  if (engram.dual_coding) {
+    if (engram.dual_coding.example) parts.push(engram.dual_coding.example);
+    if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy);
+  }
+  if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) {
+    for (const a of engram.knowledge_anchors) {
+      if (a.snippet) parts.push(a.snippet);
+    }
+  }
+  return parts.join(" ");
+}
+function embeddingContentHash(engram) {
+  return hashEmbeddedText(engramSearchText(engram));
+}
+function hashEmbeddedText(text) {
+  return createHash("md5").update(text).digest("hex");
+}
+function termMatches(t, qt) {
+  return t.includes(qt) || qt.startsWith(t);
+}
+function computeIdf(engrams, queryTokens, stats) {
+  if (stats) {
+    if (stats.N === 0) return /* @__PURE__ */ new Map();
+    const idf2 = /* @__PURE__ */ new Map();
+    for (const qt of queryTokens) {
+      const df = stats.df.get(qt) ?? 0;
+      idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df))));
+    }
+    return idf2;
+  }
+  const N = engrams.length;
+  if (N === 0) return /* @__PURE__ */ new Map();
+  const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
+  const idf = /* @__PURE__ */ new Map();
+  for (const qt of queryTokens) {
+    let df = 0;
+    for (const termSet of engramTermSets) {
+      if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) {
+        df++;
+      }
+    }
+    idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
+  }
+  return idf;
+}
+function extendCorpusStats(stats, queryTokens, outsiders) {
+  if (outsiders.length === 0) return stats;
+  const termSets = [];
+  let totalLen = 0;
+  for (const e of outsiders) {
+    const terms = ftsTokenize(engramSearchText(e));
+    totalLen += terms.length;
+    termSets.push(new Set(terms));
+  }
+  const df = new Map(stats.df);
+  for (const qt of queryTokens) {
+    let added = 0;
+    for (const set of termSets) {
+      if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++;
+    }
+    if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added);
+  }
+  const N = stats.N + outsiders.length;
+  return {
+    N,
+    df,
+    avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0
+  };
+}
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
+function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
+  const allTerms = ftsTokenize(engramSearchText(engram));
+  if (queryTokens.length === 0) return 0;
+  const docLen = allTerms.length;
+  const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
+  const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
+  let score = 0;
+  for (const qt of queryTokens) {
+    let effectiveIdf;
+    if (!idfWeights) {
+      effectiveIdf = 1;
+    } else if (hasNonZeroIdf) {
+      effectiveIdf = idfWeights.get(qt) ?? 0;
+      if (effectiveIdf === 0) continue;
+    } else {
+      effectiveIdf = 1;
+    }
+    let tf = 0;
+    for (const t of allTerms) {
+      if (termMatches(t, qt)) tf++;
+    }
+    if (tf === 0) continue;
+    const numerator = tf * (BM25_K1 + 1);
+    const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
+    score += effectiveIdf * (numerator / denominator);
+  }
+  return score;
+}
+function searchEngrams(engrams, query, limit = 20, stats) {
+  const queryTokens = ftsTokenize(query);
+  if (queryTokens.length === 0) return [];
+  const idfWeights = computeIdf(engrams, queryTokens, stats);
+  const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
+  let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0);
+  if (scored.length === 0) {
+    scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0);
+  }
+  return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
+}
+
+export {
+  MIN_TOKEN_LENGTH,
+  TOKENIZER_VERSION,
+  ftsTokenize,
+  engramSearchText,
+  embeddingContentHash,
+  hashEmbeddedText,
+  termMatches,
+  computeIdf,
+  extendCorpusStats,
+  ftsScore,
+  searchEngrams
+};

dist/chunk-W56Y5QPY.js

+import {
+  engramSearchText
+} from "./chunk-SYCJM6JJ.js";
+import {
+  atomicWrite
+} from "./chunk-TXHLQGN3.js";
+import {
+  logger
+} from "./chunk-E4YVUWMJ.js";
+
+// src/embeddings.ts
+import { existsSync, readFileSync, mkdirSync } from "fs";
+import { join, dirname } from "path";
+import { createHash } from "crypto";
+var EMBED_DIM = 384;
+var embedPipeline = null;
+var lastLoadError = null;
+var transformersUnavailable = false;
+function readDisabledFromEnv(env) {
+  const raw = env.PLUR_DISABLE_EMBEDDINGS;
+  if (!raw) return null;
+  const normalized = raw.trim().toLowerCase();
+  if (normalized === "1" || normalized === "true" || normalized === "yes") {
+    return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var";
+  }
+  return null;
+}
+var ENV_DISABLED_REASON = readDisabledFromEnv(process.env);
+var embeddingsDisabled = ENV_DISABLED_REASON !== null;
+var disabledReason = ENV_DISABLED_REASON;
+function embedderStatus() {
+  return {
+    available: !embeddingsDisabled && !transformersUnavailable,
+    loaded: embedPipeline !== null,
+    lastError: lastLoadError,
+    disabled: embeddingsDisabled,
+    disabledReason
+  };
+}
+function setEmbeddingsEnabled(enabled, reason) {
+  embeddingsDisabled = !enabled;
+  disabledReason = enabled ? null : reason ?? "embeddings disabled by config";
+  if (!enabled) {
+    embedPipeline = null;
+  }
+}
+function resetEmbedder() {
+  transformersUnavailable = false;
+  lastLoadError = null;
+  embedPipeline = null;
+}
+function _setCachedEmbedder(adapter) {
+  embedPipeline = adapter;
+  transformersUnavailable = false;
+  lastLoadError = null;
+}
+async function getEmbedder() {
+  if (embeddingsDisabled) return null;
+  if (embedPipeline) return embedPipeline;
+  try {
+    const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-TB252LRE.js");
+    const adapter = getAdapter(resolveEmbedderName());
+    embedPipeline = adapter;
+    transformersUnavailable = false;
+    lastLoadError = null;
+    return embedPipeline;
+  } catch (err) {
+    transformersUnavailable = true;
+    lastLoadError = err instanceof Error ? err.message : String(err);
+    return null;
+  }
+}
+async function embed(text, role) {
+  const embedder = await getEmbedder();
+  if (!embedder) return null;
+  if (typeof embedder.embed === "function") {
+    let vector;
+    try {
+      vector = await embedder.embed(text, role);
+    } catch (err) {
+      transformersUnavailable = true;
+      lastLoadError = err instanceof Error ? err.message : String(err);
+      embedPipeline = null;
+      return null;
+    }
+    if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) {
+      throw new Error(
+        `Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.`
+      );
+    }
+    return vector;
+  }
+  const result = await embedder(text, { pooling: "cls", normalize: true });
+  return new Float32Array(result.data);
+}
+async function getActiveEmbedderMeta() {
+  const embedder = await getEmbedder();
+  if (!embedder) return null;
+  if (typeof embedder.name === "string" && typeof embedder.dim === "number") {
+    return { name: embedder.name, dim: embedder.dim };
+  }
+  return { name: "legacy-pipeline", dim: 0 };
+}
+async function activeEmbedderDim() {
+  const meta = await getActiveEmbedderMeta();
+  return meta && meta.dim > 0 ? meta.dim : null;
+}
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+var CACHE_VERSION = 1;
+function emptyCache(meta) {
+  return {
+    meta: {
+      embedder_name: meta.name,
+      embedder_dim: meta.dim,
+      version: CACHE_VERSION
+    },
+    entries: {}
+  };
+}
+function loadCache(cachePath, active) {
+  if (!existsSync(cachePath)) return emptyCache(active);
+  try {
+    const raw = JSON.parse(readFileSync(cachePath, "utf8"));
+    if (!raw || typeof raw !== "object" || !raw.meta) {
+      logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`);
+      return emptyCache(active);
+    }
+    const meta = raw.meta;
+    if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) {
+      logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`);
+      return emptyCache(active);
+    }
+    const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {};
+    return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries };
+  } catch {
+    return emptyCache(active);
+  }
+}
+function saveCache(cachePath, cache) {
+  const dir = dirname(cachePath);
+  if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true });
+  atomicWrite(cachePath, JSON.stringify(cache), { durable: false });
+}
+function hashStatement(statement) {
+  return createHash("sha256").update(statement).digest("hex").slice(0, 16);
+}
+async function embeddingSearch(engrams, query, limit, storagePath) {
+  if (engrams.length === 0) return [];
+  const activeMeta = await getActiveEmbedderMeta();
+  if (!activeMeta) return [];
+  const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
+  const cache = loadCache(cachePath, activeMeta);
+  const queryEmbedding = await embed(query, "query");
+  if (!queryEmbedding) {
+    return [];
+  }
+  const similarities = [];
+  for (const engram of engrams) {
+    const searchText = engramSearchText(engram);
+    const hash = hashStatement(searchText);
+    let engramEmbedding;
+    if (cache.entries[engram.id]?.hash === hash) {
+      engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
+    } else {
+      const emb = await embed(searchText);
+      if (!emb) return [];
+      engramEmbedding = emb;
+      cache.entries[engram.id] = {
+        hash,
+        embedding: Array.from(engramEmbedding)
+      };
+    }
+    const score = cosineSimilarity(queryEmbedding, engramEmbedding);
+    similarities.push({ engram, score });
+  }
+  saveCache(cachePath, cache);
+  similarities.sort((a, b) => b.score - a.score);
+  return similarities.slice(0, limit).map((s) => s.engram);
+}
+async function embeddingSearchWithScores(engrams, query, limit, storagePath) {
+  if (engrams.length === 0) return [];
+  const activeMeta = await getActiveEmbedderMeta();
+  if (!activeMeta) return [];
+  const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
+  const cache = loadCache(cachePath, activeMeta);
+  const queryEmbedding = await embed(query, "query");
+  if (!queryEmbedding) {
+    return [];
+  }
+  const similarities = [];
+  for (const engram of engrams) {
+    const searchText = engramSearchText(engram);
+    const hash = hashStatement(searchText);
+    let engramEmbedding;
+    if (cache.entries[engram.id]?.hash === hash) {
+      engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
+    } else {
+      const emb = await embed(searchText);
+      if (!emb) return [];
+      engramEmbedding = emb;
+      cache.entries[engram.id] = {
+        hash,
+        embedding: Array.from(engramEmbedding)
+      };
+    }
+    const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding);
+    const score = Math.max(0, Math.min(1, rawScore));
+    similarities.push({ engram, score });
+  }
+  saveCache(cachePath, cache);
+  similarities.sort((a, b) => b.score - a.score);
+  return similarities.slice(0, limit);
+}
+async function rebuildJsonCache(engrams, storagePath, opts) {
+  const activeMeta = await getActiveEmbedderMeta();
+  if (!activeMeta) {
+    return { reembedded: 0, skipped: true, reason: "embedder unavailable" };
+  }
+  const cachePath = join(storagePath, ".embeddings-cache.json");
+  const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta);
+  let count = 0;
+  for (const engram of engrams) {
+    const searchText = engramSearchText(engram);
+    const hash = hashStatement(searchText);
+    if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue;
+    const vec = await embed(searchText);
+    if (!vec) {
+      return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" };
+    }
+    cache.entries[engram.id] = { hash, embedding: Array.from(vec) };
+    count++;
+  }
+  saveCache(cachePath, cache);
+  return { reembedded: count, skipped: false };
+}
+
+export {
+  EMBED_DIM,
+  readDisabledFromEnv,
+  embedderStatus,
+  setEmbeddingsEnabled,
+  resetEmbedder,
+  _setCachedEmbedder,
+  embed,
+  activeEmbedderDim,
+  cosineSimilarity,
+  embeddingSearch,
+  embeddingSearchWithScores,
+  rebuildJsonCache
+};

dist/embeddings-XIX5UXOX.js

+import {
+  EMBED_DIM,
+  _setCachedEmbedder,
+  activeEmbedderDim,
+  cosineSimilarity,
+  embed,
+  embedderStatus,
+  embeddingSearch,
+  embeddingSearchWithScores,
+  readDisabledFromEnv,
+  rebuildJsonCache,
+  resetEmbedder,
+  setEmbeddingsEnabled
+} from "./chunk-W56Y5QPY.js";
+import "./chunk-SYCJM6JJ.js";
+import "./chunk-TXHLQGN3.js";
+import "./chunk-E4YVUWMJ.js";
+export {
+  EMBED_DIM,
+  _setCachedEmbedder,
+  activeEmbedderDim,
+  cosineSimilarity,
+  embed,
+  embedderStatus,
+  embeddingSearch,
+  embeddingSearchWithScores,
+  readDisabledFromEnv,
+  rebuildJsonCache,
+  resetEmbedder,
+  setEmbeddingsEnabled
+};

dist/fts-AMOTYVHA.js

+import {
+  MIN_TOKEN_LENGTH,
+  TOKENIZER_VERSION,
+  computeIdf,
+  embeddingContentHash,
+  engramSearchText,
+  extendCorpusStats,
+  ftsScore,
+  ftsTokenize,
+  hashEmbeddedText,
+  searchEngrams,
+  termMatches
+} from "./chunk-SYCJM6JJ.js";
+export {
+  MIN_TOKEN_LENGTH,
+  TOKENIZER_VERSION,
+  computeIdf,
+  embeddingContentHash,
+  engramSearchText,
+  extendCorpusStats,
+  ftsScore,
+  ftsTokenize,
+  hashEmbeddedText,
+  searchEngrams,
+  termMatches
+};

package.json

 {
   "name": "@plur-ai/core",
-  "version": "0.17.1",
+  "version": "0.17.2",
   "type": "module",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

dist/chunk-SKVT6ZGO.js

-// src/fts.ts
-import { createHash } from "crypto";
-var STOP_WORDS = /* @__PURE__ */ new Set([
-  "the",
-  "and",
-  "for",
-  "that",
-  "this",
-  "with",
-  "from",
-  "are",
-  "was",
-  "were",
-  "been",
-  "have",
-  "has",
-  "not",
-  "but",
-  "its",
-  "you",
-  "your",
-  "can",
-  "will",
-  "should",
-  "would",
-  "could",
-  "may",
-  "might"
-]);
-function ftsTokenize(text) {
-  return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
-}
-function engramSearchText(engram) {
-  const parts = [engram.statement];
-  if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
-  if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
-  if (engram.entities) {
-    for (const e of engram.entities) {
-      parts.push(e.name);
-      if (e.type !== "other") parts.push(e.type);
-    }
-  }
-  if (engram.temporal) {
-    if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
-    if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
-  }
-  if (engram.rationale) parts.push(engram.rationale);
-  if (engram.source) parts.push(engram.source);
-  if (engram.dual_coding) {
-    if (engram.dual_coding.example) parts.push(engram.dual_coding.example);
-    if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy);
-  }
-  if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) {
-    for (const a of engram.knowledge_anchors) {
-      if (a.snippet) parts.push(a.snippet);
-    }
-  }
-  return parts.join(" ");
-}
-function embeddingContentHash(engram) {
-  return hashEmbeddedText(engramSearchText(engram));
-}
-function hashEmbeddedText(text) {
-  return createHash("md5").update(text).digest("hex");
-}
-function termMatches(t, qt) {
-  return t.includes(qt) || qt.startsWith(t);
-}
-function computeIdf(engrams, queryTokens, stats) {
-  if (stats) {
-    if (stats.N === 0) return /* @__PURE__ */ new Map();
-    const idf2 = /* @__PURE__ */ new Map();
-    for (const qt of queryTokens) {
-      const df = stats.df.get(qt) ?? 0;
-      idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df))));
-    }
-    return idf2;
-  }
-  const N = engrams.length;
-  if (N === 0) return /* @__PURE__ */ new Map();
-  const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
-  const idf = /* @__PURE__ */ new Map();
-  for (const qt of queryTokens) {
-    let df = 0;
-    for (const termSet of engramTermSets) {
-      if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) {
-        df++;
-      }
-    }
-    idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
-  }
-  return idf;
-}
-function extendCorpusStats(stats, queryTokens, outsiders) {
-  if (outsiders.length === 0) return stats;
-  const termSets = [];
-  let totalLen = 0;
-  for (const e of outsiders) {
-    const terms = ftsTokenize(engramSearchText(e));
-    totalLen += terms.length;
-    termSets.push(new Set(terms));
-  }
-  const df = new Map(stats.df);
-  for (const qt of queryTokens) {
-    let added = 0;
-    for (const set of termSets) {
-      if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++;
-    }
-    if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added);
-  }
-  const N = stats.N + outsiders.length;
-  return {
-    N,
-    df,
-    avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0
-  };
-}
-var BM25_K1 = 1.2;
-var BM25_B = 0.75;
-function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
-  const allTerms = ftsTokenize(engramSearchText(engram));
-  if (queryTokens.length === 0) return 0;
-  const docLen = allTerms.length;
-  const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
-  const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
-  let score = 0;
-  for (const qt of queryTokens) {
-    let effectiveIdf;
-    if (!idfWeights) {
-      effectiveIdf = 1;
-    } else if (hasNonZeroIdf) {
-      effectiveIdf = idfWeights.get(qt) ?? 0;
-      if (effectiveIdf === 0) continue;
-    } else {
-      effectiveIdf = 1;
-    }
-    let tf = 0;
-    for (const t of allTerms) {
-      if (termMatches(t, qt)) tf++;
-    }
-    if (tf === 0) continue;
-    const numerator = tf * (BM25_K1 + 1);
-    const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
-    score += effectiveIdf * (numerator / denominator);
-  }
-  return score;
-}
-function searchEngrams(engrams, query, limit = 20, stats) {
-  const queryTokens = ftsTokenize(query);
-  if (queryTokens.length === 0) return [];
-  const idfWeights = computeIdf(engrams, queryTokens, stats);
-  const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
-  let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0);
-  if (scored.length === 0) {
-    scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0);
-  }
-  return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
-}
-
-export {
-  ftsTokenize,
-  engramSearchText,
-  embeddingContentHash,
-  hashEmbeddedText,
-  termMatches,
-  computeIdf,
-  extendCorpusStats,
-  ftsScore,
-  searchEngrams
-};

dist/chunk-UND3VZDP.js

-import {
-  engramSearchText
-} from "./chunk-SKVT6ZGO.js";
-import {
-  atomicWrite
-} from "./chunk-TXHLQGN3.js";
-import {
-  logger
-} from "./chunk-E4YVUWMJ.js";
-
-// src/embeddings.ts
-import { existsSync, readFileSync, mkdirSync } from "fs";
-import { join, dirname } from "path";
-import { createHash } from "crypto";
-var EMBED_DIM = 384;
-var embedPipeline = null;
-var lastLoadError = null;
-var transformersUnavailable = false;
-function readDisabledFromEnv(env) {
-  const raw = env.PLUR_DISABLE_EMBEDDINGS;
-  if (!raw) return null;
-  const normalized = raw.trim().toLowerCase();
-  if (normalized === "1" || normalized === "true" || normalized === "yes") {
-    return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var";
-  }
-  return null;
-}
-var ENV_DISABLED_REASON = readDisabledFromEnv(process.env);
-var embeddingsDisabled = ENV_DISABLED_REASON !== null;
-var disabledReason = ENV_DISABLED_REASON;
-function embedderStatus() {
-  return {
-    available: !embeddingsDisabled && !transformersUnavailable,
-    loaded: embedPipeline !== null,
-    lastError: lastLoadError,
-    disabled: embeddingsDisabled,
-    disabledReason
-  };
-}
-function setEmbeddingsEnabled(enabled, reason) {
-  embeddingsDisabled = !enabled;
-  disabledReason = enabled ? null : reason ?? "embeddings disabled by config";
-  if (!enabled) {
-    embedPipeline = null;
-  }
-}
-function resetEmbedder() {
-  transformersUnavailable = false;
-  lastLoadError = null;
-  embedPipeline = null;
-}
-function _setCachedEmbedder(adapter) {
-  embedPipeline = adapter;
-  transformersUnavailable = false;
-  lastLoadError = null;
-}
-async function getEmbedder() {
-  if (embeddingsDisabled) return null;
-  if (embedPipeline) return embedPipeline;
-  try {
-    const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-TB252LRE.js");
-    const adapter = getAdapter(resolveEmbedderName());
-    embedPipeline = adapter;
-    transformersUnavailable = false;
-    lastLoadError = null;
-    return embedPipeline;
-  } catch (err) {
-    transformersUnavailable = true;
-    lastLoadError = err instanceof Error ? err.message : String(err);
-    return null;
-  }
-}
-async function embed(text, role) {
-  const embedder = await getEmbedder();
-  if (!embedder) return null;
-  if (typeof embedder.embed === "function") {
-    let vector;
-    try {
-      vector = await embedder.embed(text, role);
-    } catch (err) {
-      transformersUnavailable = true;
-      lastLoadError = err instanceof Error ? err.message : String(err);
-      embedPipeline = null;
-      return null;
-    }
-    if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) {
-      throw new Error(
-        `Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.`
-      );
-    }
-    return vector;
-  }
-  const result = await embedder(text, { pooling: "cls", normalize: true });
-  return new Float32Array(result.data);
-}
-async function getActiveEmbedderMeta() {
-  const embedder = await getEmbedder();
-  if (!embedder) return null;
-  if (typeof embedder.name === "string" && typeof embedder.dim === "number") {
-    return { name: embedder.name, dim: embedder.dim };
-  }
-  return { name: "legacy-pipeline", dim: 0 };
-}
-async function activeEmbedderDim() {
-  const meta = await getActiveEmbedderMeta();
-  return meta && meta.dim > 0 ? meta.dim : null;
-}
-function cosineSimilarity(a, b) {
-  let dot = 0;
-  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
-  return dot;
-}
-var CACHE_VERSION = 1;
-function emptyCache(meta) {
-  return {
-    meta: {
-      embedder_name: meta.name,
-      embedder_dim: meta.dim,
-      version: CACHE_VERSION
-    },
-    entries: {}
-  };
-}
-function loadCache(cachePath, active) {
-  if (!existsSync(cachePath)) return emptyCache(active);
-  try {
-    const raw = JSON.parse(readFileSync(cachePath, "utf8"));
-    if (!raw || typeof raw !== "object" || !raw.meta) {
-      logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`);
-      return emptyCache(active);
-    }
-    const meta = raw.meta;
-    if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) {
-      logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`);
-      return emptyCache(active);
-    }
-    const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {};
-    return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries };
-  } catch {
-    return emptyCache(active);
-  }
-}
-function saveCache(cachePath, cache) {
-  const dir = dirname(cachePath);
-  if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true });
-  atomicWrite(cachePath, JSON.stringify(cache), { durable: false });
-}
-function hashStatement(statement) {
-  return createHash("sha256").update(statement).digest("hex").slice(0, 16);
-}
-async function embeddingSearch(engrams, query, limit, storagePath) {
-  if (engrams.length === 0) return [];
-  const activeMeta = await getActiveEmbedderMeta();
-  if (!activeMeta) return [];
-  const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
-  const cache = loadCache(cachePath, activeMeta);
-  const queryEmbedding = await embed(query, "query");
-  if (!queryEmbedding) {
-    return [];
-  }
-  const similarities = [];
-  for (const engram of engrams) {
-    const searchText = engramSearchText(engram);
-    const hash = hashStatement(searchText);
-    let engramEmbedding;
-    if (cache.entries[engram.id]?.hash === hash) {
-      engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
-    } else {
-      const emb = await embed(searchText);
-      if (!emb) return [];
-      engramEmbedding = emb;
-      cache.entries[engram.id] = {
-        hash,
-        embedding: Array.from(engramEmbedding)
-      };
-    }
-    const score = cosineSimilarity(queryEmbedding, engramEmbedding);
-    similarities.push({ engram, score });
-  }
-  saveCache(cachePath, cache);
-  similarities.sort((a, b) => b.score - a.score);
-  return similarities.slice(0, limit).map((s) => s.engram);
-}
-async function embeddingSearchWithScores(engrams, query, limit, storagePath) {
-  if (engrams.length === 0) return [];
-  const activeMeta = await getActiveEmbedderMeta();
-  if (!activeMeta) return [];
-  const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
-  const cache = loadCache(cachePath, activeMeta);
-  const queryEmbedding = await embed(query, "query");
-  if (!queryEmbedding) {
-    return [];
-  }
-  const similarities = [];
-  for (const engram of engrams) {
-    const searchText = engramSearchText(engram);
-    const hash = hashStatement(searchText);
-    let engramEmbedding;
-    if (cache.entries[engram.id]?.hash === hash) {
-      engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
-    } else {
-      const emb = await embed(searchText);
-      if (!emb) return [];
-      engramEmbedding = emb;
-      cache.entries[engram.id] = {
-        hash,
-        embedding: Array.from(engramEmbedding)
-      };
-    }
-    const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding);
-    const score = Math.max(0, Math.min(1, rawScore));
-    similarities.push({ engram, score });
-  }
-  saveCache(cachePath, cache);
-  similarities.sort((a, b) => b.score - a.score);
-  return similarities.slice(0, limit);
-}
-async function rebuildJsonCache(engrams, storagePath, opts) {
-  const activeMeta = await getActiveEmbedderMeta();
-  if (!activeMeta) {
-    return { reembedded: 0, skipped: true, reason: "embedder unavailable" };
-  }
-  const cachePath = join(storagePath, ".embeddings-cache.json");
-  const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta);
-  let count = 0;
-  for (const engram of engrams) {
-    const searchText = engramSearchText(engram);
-    const hash = hashStatement(searchText);
-    if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue;
-    const vec = await embed(searchText);
-    if (!vec) {
-      return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" };
-    }
-    cache.entries[engram.id] = { hash, embedding: Array.from(vec) };
-    count++;
-  }
-  saveCache(cachePath, cache);
-  return { reembedded: count, skipped: false };
-}
-
-export {
-  EMBED_DIM,
-  readDisabledFromEnv,
-  embedderStatus,
-  setEmbeddingsEnabled,
-  resetEmbedder,
-  _setCachedEmbedder,
-  embed,
-  activeEmbedderDim,
-  cosineSimilarity,
-  embeddingSearch,
-  embeddingSearchWithScores,
-  rebuildJsonCache
-};

dist/embeddings-X7ND4OVG.js

-import {
-  EMBED_DIM,
-  _setCachedEmbedder,
-  activeEmbedderDim,
-  cosineSimilarity,
-  embed,
-  embedderStatus,
-  embeddingSearch,
-  embeddingSearchWithScores,
-  readDisabledFromEnv,
-  rebuildJsonCache,
-  resetEmbedder,
-  setEmbeddingsEnabled
-} from "./chunk-UND3VZDP.js";
-import "./chunk-SKVT6ZGO.js";
-import "./chunk-TXHLQGN3.js";
-import "./chunk-E4YVUWMJ.js";
-export {
-  EMBED_DIM,
-  _setCachedEmbedder,
-  activeEmbedderDim,
-  cosineSimilarity,
-  embed,
-  embedderStatus,
-  embeddingSearch,
-  embeddingSearchWithScores,
-  readDisabledFromEnv,
-  rebuildJsonCache,
-  resetEmbedder,
-  setEmbeddingsEnabled
-};

dist/fts-PMIWKG7S.js

-import {
-  computeIdf,
-  embeddingContentHash,
-  engramSearchText,
-  extendCorpusStats,
-  ftsScore,
-  ftsTokenize,
-  hashEmbeddedText,
-  searchEngrams,
-  termMatches
-} from "./chunk-SKVT6ZGO.js";
-export {
-  computeIdf,
-  embeddingContentHash,
-  engramSearchText,
-  extendCorpusStats,
-  ftsScore,
-  ftsTokenize,
-  hashEmbeddedText,
-  searchEngrams,
-  termMatches
-};

New alerts

AI-detected potential code anomaly

Supply chain risk

AI has identified unusual behaviors that may pose a security risk.

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Improved metrics

Total package byte prevSize

1231648

0.99%

Lines of code

28082

0.84%

Worsened metrics

Number of low supply chain risk alerts

58

1.75%

No dependency changes