New:Socket for Asana Is Now Available.Learn more
Get Started

@plur-ai/core

Package Overview
Dependencies
Maintainers
1
Versions
61
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@plur-ai/core - npm Package Compare versions

Comparing version
0.17.2
to
0.18.0
+195
dist/chunk-2UD5K4YD.js
// src/fts.ts
import { createHash } from "crypto";
var STOP_WORDS = /* @__PURE__ */ new Set([
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"are",
"was",
"were",
"been",
"have",
"has",
"not",
"but",
"its",
"you",
"your",
"can",
"will",
"should",
"would",
"could",
"may",
"might"
]);
var MIN_TOKEN_LENGTH = 2;
var TOKENIZER_VERSION = 4;
var SPACELESS_RUN = /[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Thai}\p{Script_Extensions=Khmer}\p{Script_Extensions=Lao}\p{Script_Extensions=Myanmar}]{2,}/gu;
var MAX_SPACELESS_RUN_CHARS = 512;
var DENSE_SCRIPT = /[\p{Script_Extensions=Hangul}\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}]/u;
function ftsTokenize(text) {
const lower = text.toLowerCase();
const wordSource = lower.replace(SPACELESS_RUN, " ").replace(/[^\p{L}\p{N}\p{M}_\s]/gu, " ");
const tokens = wordSource.split(/\s+/).filter((w) => w.length > 2 || w.length === 2 && DENSE_SCRIPT.test(w)).filter((w) => !STOP_WORDS.has(w));
for (const run of lower.match(SPACELESS_RUN) ?? []) {
const span = Math.min(run.length, MAX_SPACELESS_RUN_CHARS);
for (let i = 0; i < span - 1; i++) tokens.push(run.slice(i, i + 2));
}
return tokens;
}
function searchTextFrom(fields) {
return engramSearchText({
...fields,
tags: fields.tags ?? [],
domain: fields.domain ?? void 0,
rationale: fields.rationale ?? void 0,
source: fields.source ?? void 0
});
}
function engramSearchText(engram) {
const parts = [engram.statement];
if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
if (engram.entities) {
for (const e of engram.entities) {
parts.push(e.name);
if (e.type !== "other") parts.push(e.type);
}
}
if (engram.temporal) {
if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
}
if (engram.rationale) parts.push(engram.rationale);
if (engram.source) parts.push(engram.source);
if (engram.dual_coding) {
if (engram.dual_coding.example) parts.push(engram.dual_coding.example);
if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy);
}
if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) {
for (const a of engram.knowledge_anchors) {
if (a.snippet) parts.push(a.snippet);
}
}
return parts.join(" ");
}
function embeddingContentHash(engram) {
return hashEmbeddedText(engramSearchText(engram));
}
function hashEmbeddedText(text) {
return createHash("md5").update(text).digest("hex");
}
function termMatches(t, qt) {
return t.includes(qt) || qt.startsWith(t);
}
function computeIdf(engrams, queryTokens, stats) {
if (stats) {
if (stats.N === 0) return /* @__PURE__ */ new Map();
const idf2 = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
const df = stats.df.get(qt) ?? 0;
idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df))));
}
return idf2;
}
const N = engrams.length;
if (N === 0) return /* @__PURE__ */ new Map();
const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
const idf = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
let df = 0;
for (const termSet of engramTermSets) {
if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) {
df++;
}
}
idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
}
return idf;
}
function extendCorpusStats(stats, queryTokens, outsiders) {
if (outsiders.length === 0) return stats;
const termSets = [];
let totalLen = 0;
for (const e of outsiders) {
const terms = ftsTokenize(engramSearchText(e));
totalLen += terms.length;
termSets.push(new Set(terms));
}
const df = new Map(stats.df);
for (const qt of queryTokens) {
let added = 0;
for (const set of termSets) {
if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++;
}
if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added);
}
const N = stats.N + outsiders.length;
return {
N,
df,
avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0
};
}
var BM25_K1 = 1.2;
var BM25_B = 0.75;
function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
const allTerms = ftsTokenize(engramSearchText(engram));
if (queryTokens.length === 0) return 0;
const docLen = allTerms.length;
const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
let score = 0;
for (const qt of queryTokens) {
let effectiveIdf;
if (!idfWeights) {
effectiveIdf = 1;
} else if (hasNonZeroIdf) {
effectiveIdf = idfWeights.get(qt) ?? 0;
if (effectiveIdf === 0) continue;
} else {
effectiveIdf = 1;
}
let tf = 0;
for (const t of allTerms) {
if (termMatches(t, qt)) tf++;
}
if (tf === 0) continue;
const numerator = tf * (BM25_K1 + 1);
const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
score += effectiveIdf * (numerator / denominator);
}
return score;
}
function searchEngrams(engrams, query, limit = 20, stats) {
const queryTokens = ftsTokenize(query);
if (queryTokens.length === 0) return [];
const idfWeights = computeIdf(engrams, queryTokens, stats);
const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0);
if (scored.length === 0) {
scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0);
}
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
}
export {
MIN_TOKEN_LENGTH,
TOKENIZER_VERSION,
MAX_SPACELESS_RUN_CHARS,
ftsTokenize,
searchTextFrom,
engramSearchText,
embeddingContentHash,
hashEmbeddedText,
termMatches,
computeIdf,
extendCorpusStats,
ftsScore,
searchEngrams
};
import {
atomicWrite,
withLock
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/schemas/engram.ts
import { z } from "zod";
var ActivationSchema = z.object({
retrieval_strength: z.number().min(0).max(1),
storage_strength: z.number().min(0).max(1),
frequency: z.number().int().min(0),
last_accessed: z.string().describe("Date or ISO 8601 timestamp of last access.")
}).describe("ACT-R activation parameters driving decay and ranking. STABLE.");
var KnowledgeTypeSchema = z.object({
memory_class: z.enum(["semantic", "episodic", "procedural", "metacognitive"]),
cognitive_level: z.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"]).describe("Bloom's taxonomy level.")
});
var KnowledgeAnchorSchema = z.object({
path: z.string().describe("Path to a grounding document/file."),
relevance: z.enum(["primary", "supporting", "example"]).default("supporting"),
snippet: z.string().max(200).optional(),
snippet_extracted_at: z.string().optional()
});
var AssociationSchema = z.object({
target_type: z.enum(["engram", "document"]),
target: z.string().describe("ID or path of the association target."),
strength: z.number().min(0).max(0.95),
type: z.enum(["semantic", "temporal", "causal", "co_accessed"]),
updated_at: z.string().optional()
});
var DualCodingSchema = z.object({
example: z.string().optional(),
analogy: z.string().optional()
}).describe("Worked example and/or analogy (dual coding). At least one of example or analogy MUST be provided (enforced at runtime by the Zod .refine below).").refine(
(d) => d.example || d.analogy,
"At least one of example or analogy must be provided"
);
var RelationsSchema = z.object({
broader: z.array(z.string()).default([]),
narrower: z.array(z.string()).default([]),
related: z.array(z.string()).default([]),
conflicts: z.array(z.string()).default([]),
/** IDs of engrams this one intentionally replaces (#240). An intentional
* update is not a tension — the scanner skips supersedes-linked pairs. */
supersedes: z.array(z.string()).default([]),
/** Reverse edge of `supersedes` (#240) — IDs of engrams that replace this one. */
superseded_by: z.array(z.string()).default([])
}).describe("Typed graph edges between engram IDs.");
var ProvenanceSchema = z.object({
origin: z.string(),
chain: z.array(z.string()).default([]),
signature: z.string().nullable().default(null).describe("RESERVED. Detached signature over the engram. Algorithm and canonicalization not yet specified \u2014 see ENGRAM-STANDARD-v1.md \xA77."),
license: z.string().default("cc-by-sa-4.0")
}).describe("Origin and signing chain. STABLE for origin/chain/license; signature is RESERVED (see ENGRAM-STANDARD-v1.md \xA77).");
var FeedbackSignalsSchema = z.object({
positive: z.number().int().default(0),
negative: z.number().int().default(0),
neutral: z.number().int().default(0)
});
var EntityRefSchema = z.object({
name: z.string(),
type: z.enum([
"person",
"organization",
"technology",
"concept",
"project",
"tool",
"place",
"event",
"standard",
"other"
]),
uri: z.string().url().optional()
});
var TemporalSchema = z.object({
learned_at: z.string(),
valid_from: z.string().optional(),
valid_until: z.string().optional(),
ingested_at: z.string().optional()
}).describe("Bi-temporal anchoring (Zep-inspired). When is this knowledge true?");
var UsageStatsSchema = z.object({
injections: z.number().int().default(0),
hits: z.number().int().default(0),
misses: z.number().int().default(0),
last_hit_at: z.string().optional()
});
var EpisodicFieldsSchema = z.object({
emotional_weight: z.number().int().min(1).max(10).default(5),
confidence: z.number().int().min(1).max(10).default(5),
trigger_context: z.string().optional(),
journal_ref: z.string().optional()
});
var PreviousVersionRefSchema = z.object({
event_id: z.string(),
changed_at: z.string()
});
var ExchangeMetadataSchema = z.object({
fitness_score: z.number().min(0).max(1).optional(),
environmental_diversity: z.number().int().default(0),
adoption_count: z.number().int().default(0),
contradiction_rate: z.number().min(0).max(1).default(0)
});
var SerendipitySchema = z.object({
unexpectedness: z.number().min(0).max(1),
relevance: z.number().min(0).max(1),
score: z.number().min(0).max(1)
});
var InsightFateSchema = z.enum([
"surfaced",
// shown in a briefing; no downstream action yet
"promoted",
// became a durable engram / zettel
"cited",
// referenced in later journal/work
"tasked",
// converted to a GTD task
"dismissed",
// user/LLM rejected it
"expired"
// decayed out of the buffer unused
]);
var InsightFieldSchema = z.object({
/** Which memory-stream operation produced this insight. Nightly arc:
* `distill` (episode→insight synthesis) → `consolidate` (convergent gist
* abstraction over the buffer) → `dream` (divergent REM-style recombination —
* speculative, never auto-promoted). `connect`/`emerge`/`drift` are on-demand lenses. */
operation: z.enum(["distill", "consolidate", "dream", "connect", "emerge", "drift"]),
synthesized_at: z.string(),
/** Anti-hallucination grounding. Cited source notes live in the parent engram's
* `knowledge_anchors[]`; this flags whether the claim was verified against those
* snippets. `ungrounded` = couldn't cite sources → quarantined (`candidate`, never
* surfaced). `speculative` = a `dream`: its recombined INPUTS are cited but its
* CONCLUSION is an explicit hypothesis — surfaced only as inspiration, and (per the
* promote-requires-grounding refine below) it must be re-grounded to `verified`
* before it can be promoted to a durable engram. */
grounding: z.enum(["verified", "unverified", "ungrounded", "speculative"]).default("unverified"),
/** The episode-log slice this insight was distilled from (evidence trail). */
source_episode_ids: z.array(z.string()).default([]),
/** Distinct objective for connect/emerge/dream insights. */
serendipity: SerendipitySchema.optional(),
fate: InsightFateSchema.default("surfaced"),
/** Engram id / zettel path / task id the insight became, if acted upon. */
fate_ref: z.string().optional(),
fate_at: z.string().optional(),
/** How many briefings have surfaced this insight (acted-upon-rate denominator). */
surfaced_count: z.number().int().min(0).default(0)
}).refine(
// Promote-requires-grounding (user rule 2026-06-15): a dream is inspiration, not
// fact. A speculative/ungrounded insight can only become a durable promotion once
// it has been re-grounded in reality (grounding=verified).
(i) => i.fate !== "promoted" || i.grounding === "verified",
{ message: "A promoted insight must be grounded (grounding=verified); speculative dreams cannot be promoted until re-grounded.", path: ["grounding"] }
);
var ExtractionProvenanceSchema = z.object({
confidence: z.number().min(0).max(1).optional().describe("0-1 classifier confidence at extraction time. Frozen at write; distinct from feedback-derived computeConfidence() and from episodic.confidence."),
source_commit: z.string().optional().describe("Git SHA of the source repository at extraction time (reproducibility)."),
extractor_version: z.string().optional().describe("Version of the extracting CLI/tool (schema-migration handle). Complementary to the pack-level capsule producer field (#61).")
}).passthrough().describe("ETL extraction provenance convention carried in structured_data.extraction (#463). Not wired into EngramSchema.");
function getExtractionProvenance(engram) {
const extraction = engram.structured_data?.["extraction"];
if (extraction === void 0 || extraction === null) return null;
const parsed = ExtractionProvenanceSchema.safeParse(extraction);
return parsed.success ? parsed.data : null;
}
var MeasuredUnderSchema = z.object({
/** Model or system variant under which the measurement was taken (e.g. 'claude-opus-4', 'gpt-4o'). */
model: z.string().optional(),
/** Source environment type (e.g. 'local-git', 'gitlab', 'bench', 'production'). */
source_type: z.string().optional(),
/** Hardware or runtime tier (e.g. 'M3-Pro-36GB', 'A100', 'CI-runner'). */
hardware: z.string().optional(),
/** Dataset or workload identifier (e.g. 'LongMemEval-S', 'plur-bench-2026-Q2'). */
dataset: z.string().optional(),
/** ISO date (YYYY-MM-DD) the measurement was taken. */
date: z.string().optional()
}).passthrough().describe("Context under which a numeric or benchmark-derived measurement was taken (#869). All fields optional \u2014 absent means unknown.");
var EngramSchema = z.object({
// Identity
id: z.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/).describe("Unique identifier. Class prefix ENG (concrete engram), ABS (abstraction), or META (meta-engram). Canonical concrete form: ENG-YYYY-MMDD-NNN; store-namespaced form: ENG-{PREFIX}-YYYY-MMDD-NNN."),
version: z.number().int().min(1).default(2).describe("Schema-shape generation of this engram object (currently 2). Distinct from engram_version, which tracks content evolution."),
// 'active' and 'retired' are the two states any current code path assigns
// (retire via forget/dedup/supersede). 'dormant' and 'candidate' are NOT
// assigned by any code today: 'dormant' was only ever set by the batchDecay
// pass removed in #563 (decay is now a read-time property, not a materialized
// status), and 'candidate' is reserved. They are kept in the enum so stores
// written before #563 that persisted status:'dormant' still load, and so the
// status filter accepts them; do not remove without a data migration.
status: z.enum(["active", "dormant", "retired", "candidate"]).describe("Lifecycle state. Assigned values today are active/retired; dormant/candidate are legacy/reserved (see note above)."),
consolidated: z.boolean().default(false).describe("Whether this engram has been through consolidation (sleep-like batch reprocessing)."),
type: z.enum(["behavioral", "terminological", "procedural", "architectural"]).describe("Top-level classification of the knowledge."),
scope: z.string().describe("Hierarchical namespace, e.g. 'global', 'project:my-app', 'group:plur/test'. Free-form string; ':' separates scope kind from path."),
visibility: z.enum(["private", "public", "template"]).default("private").describe("Sharing posture. 'private' engrams MUST NOT be exported in packs."),
// Content
statement: z.string().min(1).describe("The assertion itself \u2014 the load-bearing content of the engram."),
rationale: z.string().optional().describe("Why this is true / why it matters."),
contraindications: z.array(z.string()).optional().describe("Conditions under which the statement does NOT apply."),
// Lineage
source: z.string().optional().describe("Free-text origin (session, document, conversation)."),
source_patterns: z.array(z.string()).optional().describe("Pattern IDs that contributed to this engram."),
derivation_count: z.number().int().min(0).default(1).describe("How many derivation steps produced this engram."),
pack: z.string().nullable().default(null).describe("Name of the pack this engram belongs to, or null."),
abstract: z.string().nullable().default(null).describe("ID of an ABS- abstraction this engram instantiates, or null."),
derived_from: z.string().nullable().default(null).describe("ID of the engram this was derived from, or null."),
// Classification
knowledge_type: KnowledgeTypeSchema.optional(),
domain: z.string().optional().describe("Dotted domain path, e.g. 'dev/testing' or 'plur.session'."),
tags: z.array(z.string()).default([]).describe("Free-form tags used for matching and retrieval."),
// Activation (ACT-R model)
activation: ActivationSchema.default({
retrieval_strength: 0.7,
storage_strength: 1,
frequency: 0,
last_accessed: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
}),
// Relations & grounding
relations: RelationsSchema.optional(),
associations: z.array(AssociationSchema).default([]),
knowledge_anchors: z.array(KnowledgeAnchorSchema).default([]),
dual_coding: DualCodingSchema.optional(),
// Provenance
provenance: ProvenanceSchema.optional(),
// Feedback
feedback_signals: FeedbackSignalsSchema.default({ positive: 0, negative: 0, neutral: 0 }),
// === NEW OPTIONAL FIELDS (v2.1) ===
/** Typed entity references extracted from statement. Enables graph queries. */
entities: z.array(EntityRefSchema).optional().describe("Typed entity references extracted from statement. Enables graph queries."),
/** Temporal validity window. When is this knowledge true? */
temporal: TemporalSchema.optional(),
/** Automatic usage tracking. Injections, hits, misses. */
usage: UsageStatsSchema.optional(),
/** Episodic context: emotional weight, confidence, trigger. */
episodic: EpisodicFieldsSchema.optional(),
/** Exchange marketplace metadata: fitness, adoption, diversity. */
exchange: ExchangeMetadataSchema.optional(),
/** Extensible key-value data for domain-specific fields. */
structured_data: z.record(z.string(), z.unknown()).optional().describe("Extensible key-value data for domain-specific fields."),
/** Memory-stream insight provenance (metacognition Phase 1). Orthogonal to
* `type`. Present iff this engram was synthesized by the metacognition memory
* stream; the episodic insight buffer is the set of engrams where this is set. */
insight: InsightFieldSchema.optional(),
/** Polarity classification: 'do' for directives, 'dont' for prohibitions, null for unclassified. */
polarity: z.enum(["do", "dont"]).nullable().default(null).describe("'do' for directives, 'dont' for prohibitions, null for unclassified."),
// === SP1: Memory Intelligence fields ===
content_hash: z.string().optional().describe("Hash of normalized statement content, used for dedup."),
commitment: z.enum(["exploring", "leaning", "decided", "locked", "draft"]).optional().describe("Commitment level of the asserted knowledge. `draft` marks an engram as pending human approval; core stores and recalls it like any other value \u2014 enforcement is left to deployments that implement a review queue. A positive feedback signal does not advance it (see feedback.ts:nextCommitment)."),
locked_at: z.string().optional().describe("Timestamp when commitment reached 'locked'."),
locked_reason: z.string().optional().describe("Why this engram was locked."),
// === SP1: Reference counting (issue #107, renamed #866) ===
/** Number of learn() write attempts that resolved to this engram.
* Incremented on every hash-dedup hit; decremented by forget().
* Engram physically retires only when this reaches 0.
* Renamed from reference_count (#866) — backfill runs on first parse of old stores. */
write_count: z.number().int().min(0).default(1).describe("Number of learn() write attempts that resolved to this engram (same-scope re-learns). Engram retires only when this reaches 0. Renamed from reference_count (#866)."),
/** Number of times this engram was selected into a session's injection context.
* Distinct from activation.frequency (which counts recall() retrieval events, not
* inject() selections). High injection_count + low feedback_signals.positive is a
* signal for #865 efficacy detection. Incremented in the inject path (#866). */
injection_count: z.number().int().min(0).default(0).describe("Number of times this engram was injected into session context. Distinct from activation.frequency (recall events). Use with feedback_signals for efficacy auditing (#866)."),
/** Provenance of each write attempt. One entry per write (including the
* first). Migrated old engrams without this field start with []. */
sources: z.array(z.object({
scope: z.string(),
session_id: z.string().nullable().default(null),
stored_at: z.string().describe("ISO 8601 timestamp of this write.")
})).default([]).describe("Provenance of each write attempt; one entry per write."),
// === SP1: Cross-scope recurrence (issue #176) ===
/** Number of times this engram's content was re-learned at a DIFFERENT
* scope than the original. Triggers auto-broadening + commitment
* escalation when threshold is crossed. Distinct from write_count
* (which counts re-learns in the SAME scope) — recurrence_count is
* evidence of universal applicability, not just repetition. */
recurrence_count: z.number().int().min(0).default(0).describe("Number of times this content was re-learned at a DIFFERENT scope than the original. Evidence of universal applicability."),
// === SP2: History & Evolution fields ===
engram_version: z.number().int().min(1).default(1).describe("Content-evolution version (incremented when the statement materially changes)."),
previous_version_ref: PreviousVersionRefSchema.optional(),
episode_ids: z.array(z.string()).default([]).describe("IDs of episodes (raw conversational events) that produced or reinforced this engram."),
// === SP3: Retrieval & Injection fields ===
summary: z.string().max(80).optional().describe("Short (<=80 char) injection-friendly summary."),
/**
* Always-load flag. Pinned engrams bypass the term-hits gate in scoreEngram
* and are eligible for injection on every session start, regardless of
* keyword overlap with the user's task. Use sparingly: meta-rules,
* cross-cutting safety conventions, and core operating principles only.
* Pinned engrams still respect the token budget — they bypass per-pack and
* per-domain fairness caps in fillTokenBudget so always-load behavior is
* honored even if a single pack contributes many.
*/
pinned: z.boolean().optional().describe("Always-load flag. Pinned engrams bypass the keyword-relevance gate and are eligible for injection every session. Use sparingly."),
/** Measurement context for numeric or benchmark-derived claims (#869).
* Records model, source_type, hardware, dataset, and/or date under which the
* asserted value was measured, so differing-condition measurements can be
* stored as refinements rather than tensions (#203). Absent for non-numeric
* engrams; all sub-fields are optional even when the object is present. */
measured_under: MeasuredUnderSchema.optional()
});
var EngramSchemaPassthrough = EngramSchema.passthrough();
// src/backup.ts
import * as fs from "fs";
import * as path from "path";
import { createHash } from "crypto";
import * as yaml from "js-yaml";
var BACKUP_DIR = "backups";
var KEEP_DAILY = 7;
var KEEP_WEEKLY = 4;
var SHRINK_TOLERANCE = 0.1;
function statePath(root) {
return path.join(root, BACKUP_DIR, ".state.json");
}
function readState(root) {
try {
return JSON.parse(fs.readFileSync(statePath(root), "utf8"));
} catch {
return {};
}
}
function writeState(root, state) {
const p = statePath(root);
fs.mkdirSync(path.dirname(p), { recursive: true });
fs.writeFileSync(p, JSON.stringify(state, null, 2) + "\n", "utf8");
}
function sha256(content) {
return createHash("sha256").update(content).digest("hex");
}
function validateStore(filePath, lastGoodCount) {
const failures = [];
const reasons = [];
let raw;
try {
raw = fs.readFileSync(filePath);
} catch (err) {
return { ok: false, failures: ["unreadable"], reasons: [`cannot read ${filePath}: ${err}`], count: null };
}
if (raw.length === 0) {
return { ok: false, failures: ["empty"], reasons: ["file is 0 bytes"], count: null };
}
if (!raw.toString("utf8").endsWith("\n")) {
failures.push("truncated");
reasons.push("file does not end with a newline \u2014 PLUR's writer always emits one, so this looks cut short");
}
let doc;
try {
doc = yaml.load(raw.toString("utf8"));
} catch (err) {
return { ok: false, failures: ["unparseable"], reasons: [`YAML parse failed: ${err}`], count: null };
}
if (doc == null || typeof doc !== "object" || Array.isArray(doc) || !Array.isArray(doc.engrams)) {
return {
ok: false,
failures: ["not-a-store"],
reasons: ["parsed, but is not a mapping with an `engrams` list"],
count: null
};
}
const entries = doc.engrams;
const count = entries.length;
let invalid = 0;
const ids = /* @__PURE__ */ new Set();
let duplicateIds = 0;
let missingIds = 0;
for (const entry of entries) {
if (!EngramSchemaPassthrough.safeParse(entry).success) invalid++;
const id = entry?.id;
if (typeof id !== "string" || id.length === 0) missingIds++;
else if (ids.has(id)) duplicateIds++;
else ids.add(id);
}
if (invalid > 0) {
failures.push("invalid-entries");
reasons.push(`${invalid} entry/entries fail schema validation`);
}
if (missingIds > 0) {
failures.push("missing-ids");
reasons.push(`${missingIds} entry/entries have no id`);
}
if (duplicateIds > 0) {
failures.push("duplicate-ids");
reasons.push(`${duplicateIds} duplicate id(s)`);
}
if (typeof lastGoodCount === "number" && lastGoodCount > 0) {
const floor = lastGoodCount * (1 - SHRINK_TOLERANCE);
if (count < floor) {
failures.push("shrunk");
reasons.push(
`holds ${count} engram(s) but the last good snapshot held ${lastGoodCount} \u2014 a drop this large is how a truncation looks`
);
}
}
return { ok: failures.length === 0, failures, reasons, count };
}
function todayStamp(now) {
return now.toISOString().slice(0, 10);
}
function snapshotPath(root, stamp) {
return path.join(root, BACKUP_DIR, `engrams-${stamp}.yaml`);
}
var doneThisProcess = /* @__PURE__ */ new Set();
function maybeDailyBackup(root, storePath, now = /* @__PURE__ */ new Date()) {
const key = `${root}\0${todayStamp(now)}`;
if (doneThisProcess.has(key)) return { taken: false, skipped: "already-today" };
try {
if (!fs.existsSync(storePath)) {
doneThisProcess.add(key);
return { taken: false, skipped: "no-store" };
}
const state = readState(root);
const stamp = todayStamp(now);
if (state.last_backup_date === stamp) {
doneThisProcess.add(key);
return { taken: false, skipped: "already-today" };
}
const existing = listBackups(root);
const strongest = existing.reduce(
(max, b) => typeof b.count === "number" && (max === void 0 || b.count > max) ? b.count : max,
void 0
);
const baseline = state.last_good_count ?? strongest;
const validity = validateStore(storePath, baseline);
if (!validity.ok) {
logger.warning(
`[plur:backup] refusing to snapshot ${storePath} \u2014 ${validity.reasons.join("; ")}. Your last good backup is unchanged. Run 'plur doctor' to inspect.`
);
return { taken: false, skipped: "invalid", validity };
}
const bytes = fs.readFileSync(storePath);
const dest = snapshotPath(root, stamp);
const sameDay = existing.find((b) => b.stamp === stamp);
if (sameDay && typeof sameDay.count === "number" && (validity.count ?? 0) < sameDay.count) {
logger.warning(
`[plur:backup] keeping today's existing snapshot (${sameDay.count} engrams) \u2014 the live store holds ${validity.count}, and replacing a stronger snapshot with a weaker one would discard the better copy. Run 'plur restore --list' to inspect.`
);
doneThisProcess.add(key);
return { taken: false, skipped: "invalid", validity };
}
fs.mkdirSync(path.dirname(dest), { recursive: true });
writeFileDurable(dest, bytes);
writeFileDurable(
`${dest}.sha256`,
Buffer.from(
`${sha256(bytes)} ${path.basename(dest)}
${validity.count} engrams
taken_at ${now.toISOString()}
`,
"utf8"
)
);
doneThisProcess.add(key);
writeState(root, {
last_backup_date: stamp,
last_good_count: validity.count ?? void 0,
last_good_sha256: sha256(bytes)
});
rotate(root, now);
return { taken: true, path: dest, validity };
} catch (err) {
logger.warning(`[plur:backup] snapshot failed (the write itself was unaffected): ${err}`);
return { taken: false, skipped: "invalid" };
}
}
function flushFileAt(filePath) {
let fd;
try {
fd = fs.openSync(filePath, "r+");
fs.fsyncSync(fd);
} catch {
} finally {
if (fd !== void 0) {
try {
fs.closeSync(fd);
} catch {
}
}
}
}
function writeFileDurable(dest, bytes) {
const fd = fs.openSync(dest, "w");
try {
fs.writeFileSync(fd, bytes);
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
}
function listBackups(root) {
const dir = path.join(root, BACKUP_DIR);
if (!fs.existsSync(dir)) return [];
const out = [];
for (const name of fs.readdirSync(dir)) {
const m = name.match(/^engrams-(\d{4}-\d{2}-\d{2})\.yaml$/);
if (!m) continue;
const full = path.join(dir, name);
const entry = { path: full, stamp: m[1], size: fs.statSync(full).size };
try {
const sidecar = fs.readFileSync(`${full}.sha256`, "utf8");
entry.sha256 = sidecar.split(/\s+/)[0];
const cm = sidecar.match(/(\d+) engrams/);
if (cm) entry.count = parseInt(cm[1], 10);
const tm = sidecar.match(/taken_at (\S+)/);
if (tm) entry.takenAt = tm[1];
} catch {
}
out.push(entry);
}
return out.sort((a, b) => a.stamp < b.stamp ? 1 : -1);
}
function rotate(root, now) {
const all = listBackups(root);
if (all.length <= KEEP_DAILY) return;
const keep = /* @__PURE__ */ new Set();
for (const b of all.slice(0, KEEP_DAILY)) keep.add(b.path);
const weeksSeen = /* @__PURE__ */ new Set();
for (const b of all.slice(KEEP_DAILY)) {
const week = isoWeek(/* @__PURE__ */ new Date(`${b.stamp}T00:00:00Z`));
if (weeksSeen.has(week)) continue;
weeksSeen.add(week);
if (weeksSeen.size <= KEEP_WEEKLY) keep.add(b.path);
}
for (const b of all) {
if (keep.has(b.path)) continue;
try {
fs.unlinkSync(b.path);
fs.unlinkSync(`${b.path}.sha256`);
} catch {
}
}
void now;
}
function isoWeek(d) {
const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
const day = t.getUTCDay() || 7;
t.setUTCDate(t.getUTCDate() + 4 - day);
const yearStart = new Date(Date.UTC(t.getUTCFullYear(), 0, 1));
const week = Math.ceil(((t.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
return `${t.getUTCFullYear()}-W${week}`;
}
function planRestore(root, storePath, stamp) {
const all = listBackups(root);
if (all.length === 0) throw new Error(`[plur] no backups found in ${path.join(root, BACKUP_DIR)}`);
const backup = stamp ? all.find((b) => b.stamp === stamp) : all[0];
if (!backup) throw new Error(`[plur] no backup for ${stamp}. Available: ${all.map((b) => b.stamp).join(", ")}`);
const bytes = fs.readFileSync(backup.path);
const actualSha256 = sha256(bytes);
const integrityOk = backup.sha256 === void 0 ? false : backup.sha256 === actualSha256;
const validity = validateStore(backup.path);
const backupIds = new Set(idsIn(backup.path));
const currentIds = idsIn(storePath);
const wouldLose = currentIds.filter((id) => !backupIds.has(id));
return {
backup,
validity,
actualSha256,
integrityOk,
wouldLose,
// Compare against the snapshot's INSTANT where we have it. Falling back to
// the end of its day is the conservative direction when a sidecar predates
// this field: it under-reports rather than inventing losses.
unrecoverable: idsCreatedAfter(root, backup.takenAt ?? `${backup.stamp}T23:59:59.999Z`).filter((id) => !backupIds.has(id))
};
}
function idsIn(filePath) {
try {
const doc = yaml.load(fs.readFileSync(filePath, "utf8"));
if (!doc || !Array.isArray(doc.engrams)) return [];
return doc.engrams.map((e) => e?.id).filter((id) => typeof id === "string");
} catch {
return [];
}
}
function idsCreatedAfter(root, since) {
const dir = path.join(root, "history");
if (!fs.existsSync(dir)) return [];
const ids = [];
for (const name of fs.readdirSync(dir)) {
if (!name.endsWith(".jsonl")) continue;
let lines;
try {
lines = fs.readFileSync(path.join(dir, name), "utf8").split("\n");
} catch {
continue;
}
for (const line of lines) {
if (!line.trim()) continue;
try {
const ev = JSON.parse(line);
if (typeof ev?.timestamp !== "string" || ev.timestamp <= since) continue;
if (typeof ev?.engram_id === "string") ids.push(ev.engram_id);
} catch {
}
}
}
return [...new Set(ids)];
}
function restoreBackup(root, storePath, opts = {}) {
let plan;
const superseded = `${storePath}.superseded-${Date.now()}`;
withLock(storePath, () => {
plan = planRestore(root, storePath, opts.stamp);
if (!opts.force) {
const problems = [];
if (!plan.validity.ok) problems.push(...plan.validity.reasons);
if (!plan.integrityOk) {
problems.push(
plan.backup.sha256 === void 0 ? "no sha256 sidecar \u2014 cannot verify the backup is intact" : "sha256 does not match the sidecar \u2014 the backup itself is damaged"
);
}
if (problems.length > 0) {
throw new Error(
`[plur] refusing to restore ${plan.backup.path}: ${problems.join("; ")}.
Restoring is a whole-corpus overwrite; doing it from a backup that does not verify would replace a damaged store with a differently damaged one.
Pass force to override if you have inspected the file yourself.`
);
}
}
if (fs.existsSync(storePath)) {
fs.copyFileSync(storePath, superseded);
flushFileAt(superseded);
}
atomicWrite(storePath, fs.readFileSync(plan.backup.path, "utf8"));
});
if (plan.wouldLose.length > 0) {
logger.warning(
`[plur:restore] ${plan.wouldLose.length} engram(s) present before the restore are not in this backup: ${plan.wouldLose.slice(0, 10).join(", ")}${plan.wouldLose.length > 10 ? ", \u2026" : ""}. The pre-restore store was kept at ${superseded}.`
);
}
if (plan.unrecoverable.length > 0) {
logger.warning(
`[plur:restore] history records ${plan.unrecoverable.length} engram(s) created after this backup that it does not contain: ${plan.unrecoverable.slice(0, 10).join(", ")}${plan.unrecoverable.length > 10 ? ", \u2026" : ""}.`
);
}
return { ...plan, restored: true, supersededPath: superseded };
}
// src/history.ts
import * as fs2 from "fs";
import { join as join2 } from "path";
import { createHash as createHash2 } from "crypto";
function appendHistory(root, event) {
const historyDir = join2(root, "history");
if (!fs2.existsSync(historyDir)) {
fs2.mkdirSync(historyDir, { recursive: true });
}
const date = event.timestamp.slice(0, 7);
const filePath = join2(historyDir, `${date}.jsonl`);
const line = JSON.stringify(event) + "\n";
try {
const fd = fs2.openSync(filePath, "a");
try {
fs2.writeSync(fd, line);
try {
fs2.fsyncSync(fd);
} catch {
}
} finally {
fs2.closeSync(fd);
}
} catch (err) {
if (!warnedHistoryPaths.has(filePath)) {
warnedHistoryPaths.add(filePath);
logger.warning(
`[plur] history could not be written to ${filePath}: ${err.message}. The operation itself succeeded. While this persists, \`plur restore\` cannot name unrecoverable engrams and engram-id allocation loses its cross-compaction guarantee (#816).`
);
}
}
}
var warnedHistoryPaths = /* @__PURE__ */ new Set();
function readHistory(root, yearMonth) {
const filePath = join2(root, "history", `${yearMonth}.jsonl`);
if (!fs2.existsSync(filePath)) return [];
const content = fs2.readFileSync(filePath, "utf8");
const lines = content.split("\n").filter((l) => l.trim().length > 0);
const events = [];
for (const line of lines) {
try {
events.push(JSON.parse(line));
} catch {
}
}
return events;
}
function mintedIdsWithPrefix(root, yearMonth, prefixes) {
try {
const out = [];
for (const ev of readHistory(root, yearMonth)) {
if (ev.event !== "engram_created") continue;
const id = ev.engram_id;
if (typeof id === "string" && prefixes.some((p) => id.startsWith(p))) out.push(id);
}
return out;
} catch {
return [];
}
}
function listHistoryMonths(root) {
const historyDir = join2(root, "history");
if (!fs2.existsSync(historyDir)) return [];
return fs2.readdirSync(historyDir).filter((f) => f.endsWith(".jsonl")).map((f) => f.replace(".jsonl", "")).sort();
}
function readHistoryForEngram(root, engramId) {
const months = listHistoryMonths(root);
const events = [];
for (const month of months) {
const monthEvents = readHistory(root, month);
for (const event of monthEvents) {
if (event.engram_id === engramId) {
events.push(event);
}
}
}
return events;
}
var _PROC_SALT = (process.pid % 1296).toString(36).padStart(2, "0");
var _evtSeq = 0;
var _injSeq = 0;
function generateEventId() {
return `EVT-${Date.now()}-${_PROC_SALT}${(_evtSeq++).toString(36).padStart(4, "0")}`;
}
function generateInjectionId() {
return `INJ-${Date.now()}-${_PROC_SALT}${(_injSeq++).toString(36).padStart(4, "0")}`;
}
function computeQueryHash(task) {
const normalized = task.toLowerCase().replace(/\s+/g, " ").trim();
return createHash2("sha256").update(normalized).digest("hex").slice(0, 16);
}
function findLatestInjectionFor(root, engramId, maxMonths = 2) {
const now = /* @__PURE__ */ new Date();
const allowed = /* @__PURE__ */ new Set();
for (let i = 0; i < maxMonths; i++) {
const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1));
allowed.add(d.toISOString().slice(0, 7));
}
const months = listHistoryMonths(root).filter((m) => allowed.has(m)).reverse();
for (const month of months) {
let latest = null;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const ids = event.data.ids;
if (!Array.isArray(ids) || !ids.includes(engramId)) continue;
if (!latest || event.timestamp > latest.timestamp) latest = event;
}
if (latest) return { injection_id: latest.engram_id, timestamp: latest.timestamp };
}
return null;
}
var INJECTION_SOURCES = /* @__PURE__ */ new Set([
"session_start",
"inject",
"hook",
"unknown"
]);
var ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
function readCoInjections(root, months) {
const events = [];
let skipped = 0;
const wanted = months ? new Set(months) : null;
for (const month of listHistoryMonths(root)) {
if (wanted && !wanted.has(month)) continue;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const raw = event.data;
if (!Array.isArray(raw.ids) || typeof raw.query_hash !== "string") {
skipped++;
continue;
}
if (typeof event.timestamp !== "string" || !ISO_TIMESTAMP.test(event.timestamp)) {
skipped++;
continue;
}
const ids = raw.ids.filter((id) => typeof id === "string" && id.length > 0);
if (ids.length !== raw.ids.length) skipped++;
const data = { ids, query_hash: raw.query_hash };
if (typeof raw.tokens_used === "number" && Number.isFinite(raw.tokens_used)) {
data.tokens_used = raw.tokens_used;
}
if (raw.source !== void 0) {
data.source = INJECTION_SOURCES.has(raw.source) ? raw.source : "unknown";
}
if (typeof raw.scope === "string") data.scope = raw.scope;
if (typeof raw.session_id === "string") data.session_id = raw.session_id;
events.push({ injection_id: event.engram_id, timestamp: event.timestamp, data });
}
}
events.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
return { events, skipped };
}
function countInjectionEvents(root) {
const counts = {
co_injection: 0,
injection_outcome: 0,
outcome_positive: 0,
outcome_negative: 0
};
for (const month of listHistoryMonths(root)) {
for (const event of readHistory(root, month)) {
if (event.event === "co_injection") {
counts.co_injection++;
} else if (event.event === "injection_outcome") {
counts.injection_outcome++;
if (event.data.signal === "positive") counts.outcome_positive++;
else if (event.data.signal === "negative") counts.outcome_negative++;
}
}
}
return counts;
}
// src/content-hash.ts
import { createHash as createHash3 } from "crypto";
var NON_WORD = /[^\p{L}\p{N}\p{M}_\s]/gu;
function normalizeStatement(statement) {
return statement.toLowerCase().replace(NON_WORD, "").replace(/\s+/g, " ").trim();
}
function computeContentHash(statement) {
const normalized = normalizeStatement(statement);
return createHash3("sha256").update(normalized).digest("hex");
}
function isHashable(statement) {
return normalizeStatement(statement).length > 0;
}
// src/dedup.ts
function buildDedupPrompt(newStatement, candidates) {
const candidateList = candidates.map(
(c, i) => `${i + 1}. [${c.id}] (${c.type}${c.domain ? ", domain: " + c.domain : ""})
"${c.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare a new memory statement against existing ones.
NEW STATEMENT:
"${newStatement}"
EXISTING ENGRAMS:
${candidateList}
For each existing engram, answer:
1. RELATIONSHIP: Is the new statement a DUPLICATE (same meaning), EVOLUTION (updated version of same knowledge), COMPLEMENTARY (related but different angle), or UNRELATED?
2. RICHNESS: Does the new statement contain more specific, actionable information than the existing one? (yes/no)
Then give your OVERALL DECISION (exactly one):
- NOOP: New statement is an exact duplicate of an existing engram. Return the ID.
- UPDATE: New statement is an evolution with MORE information. Return the ID to update.
- MERGE: New statement and an existing one are complementary \u2014 combining them preserves both. Return the ID to merge with.
- ADD: New statement is genuinely new knowledge.
Respond in this exact format:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID if UPDATE/MERGE/NOOP, or "none" if ADD>
REASON: <one sentence explanation>`;
}
function buildBatchDedupPrompt(statements, existingEngrams) {
const stmtList = statements.map((s, i) => `${i + 1}. "${s}"`).join("\n");
const engramList = existingEngrams.map(
(e, i) => `${i + 1}. [${e.id}] (${e.type}${e.domain ? ", domain: " + e.domain : ""})
"${e.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare NEW statements against existing engrams.
NEW STATEMENTS:
${stmtList}
EXISTING ENGRAMS:
${engramList}
For each NEW statement, decide:
- NOOP: Exact duplicate of an existing engram.
- UPDATE: Evolution with more info than existing.
- MERGE: Complementary with existing \u2014 combine.
- ADD: Genuinely new knowledge.
Respond with one block per new statement:
STATEMENT_1:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID or "none">
STATEMENT_2:
...`;
}
function parseDedupResponse(response) {
const decisionMatch = response.match(/DECISION:\s*(ADD|UPDATE|MERGE|NOOP)/i);
const targetMatch = response.match(/TARGET:\s*([^\n]+)/i);
const reasonMatch = response.match(/REASON:\s*([^\n]+)/i);
const decision = decisionMatch?.[1]?.toUpperCase() ?? "ADD";
const targetRaw = targetMatch?.[1]?.trim() ?? "none";
const target_id = targetRaw === "none" ? null : targetRaw.replace(/[^A-Za-z0-9-]/g, "");
const reason = reasonMatch?.[1]?.trim() ?? "";
return { decision, target_id, reason };
}
export {
ExtractionProvenanceSchema,
getExtractionProvenance,
EngramSchemaPassthrough,
BACKUP_DIR,
validateStore,
maybeDailyBackup,
listBackups,
planRestore,
restoreBackup,
appendHistory,
readHistory,
mintedIdsWithPrefix,
listHistoryMonths,
readHistoryForEngram,
generateEventId,
generateInjectionId,
computeQueryHash,
findLatestInjectionFor,
readCoInjections,
countInjectionEvents,
normalizeStatement,
computeContentHash,
isHashable,
buildDedupPrompt,
buildBatchDedupPrompt,
parseDedupResponse
};
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/embedders/transformers-base.ts
var pipelineCache = /* @__PURE__ */ new Map();
async function loadPipeline(modelId, dtype) {
const key = `${modelId}::${dtype ?? "fp32"}`;
let pending = pipelineCache.get(key);
if (!pending) {
pending = (async () => {
process.env.HF_HUB_DISABLE_XET ??= "1";
const transformers = await import("@huggingface/transformers");
const cacheDir = process.env.PLUR_MODEL_CACHE_DIR || process.env.HF_HOME;
if (cacheDir) {
;
transformers.env.cacheDir = cacheDir;
}
return transformers.pipeline("feature-extraction", modelId, dtype ? { dtype } : void 0);
})();
pipelineCache.set(key, pending);
}
return await pending;
}
function makeTransformersAdapter(config) {
const pooling = config.pooling;
const normalize = config.normalize ?? true;
async function embedOne(text) {
const pipe = await loadPipeline(config.modelId, config.dtype);
const result = await pipe(text, { pooling, normalize });
const arr = result.data instanceof Float32Array ? result.data : new Float32Array(result.data);
if (arr.length !== config.dim) {
throw new Error(
`Embedder "${config.name}" returned ${arr.length}-dim vector, expected ${config.dim}`
);
}
return arr;
}
return {
name: config.name,
dim: config.dim,
modelId: config.modelId,
embed: embedOne,
async embedBatch(texts) {
const out = [];
for (const t of texts) out.push(await embedOne(t));
return out;
}
};
}
// src/embedders/minilm.ts
var MINILM_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
function makeMiniLMAdapter() {
return makeTransformersAdapter({
name: "minilm",
dim: 384,
modelId: MINILM_MODEL_ID,
pooling: "mean",
normalize: true,
dtype: "fp32"
});
}
// src/embedders/bge-small.ts
var BGE_SMALL_MODEL_ID = "Xenova/bge-small-en-v1.5";
function makeBgeSmallAdapter() {
return makeTransformersAdapter({
name: "bge-small",
dim: 384,
modelId: BGE_SMALL_MODEL_ID,
pooling: "cls",
normalize: true,
dtype: "fp32"
});
}
// src/embedders/bge-base.ts
var BGE_BASE_MODEL_ID = "Xenova/bge-base-en-v1.5";
function makeBgeBaseAdapter() {
return makeTransformersAdapter({
name: "bge-base",
dim: 768,
modelId: BGE_BASE_MODEL_ID,
pooling: "cls",
normalize: true,
dtype: "fp32"
});
}
// src/embedders/embedding-gemma.ts
var EMBEDDING_GEMMA_MODEL_ID = "onnx-community/embeddinggemma-300m-ONNX";
var DIM = 768;
var loaded = null;
async function load() {
if (!loaded) {
loaded = (async () => {
process.env.HF_HUB_DISABLE_XET ??= "1";
const { AutoTokenizer, AutoModel } = await import("@huggingface/transformers");
const tokenizer = await AutoTokenizer.from_pretrained(EMBEDDING_GEMMA_MODEL_ID);
const model = await AutoModel.from_pretrained(EMBEDDING_GEMMA_MODEL_ID, { dtype: "q8" });
return { tokenizer, model };
})();
}
return loaded;
}
function makeEmbeddingGemmaAdapter() {
async function embedOne(text, role) {
const prefix = role === "query" ? "task: search result | query: " : "title: none | text: ";
const { tokenizer, model } = await load();
const inputs = await tokenizer(prefix + text, { padding: true, truncation: true });
const outputs = await model(inputs);
const raw = outputs.sentence_embedding.data;
const arr = raw instanceof Float32Array ? raw : new Float32Array(raw);
if (arr.length !== DIM) {
throw new Error(`EmbeddingGemma: expected ${DIM}-dim sentence_embedding, got ${arr.length}`);
}
return arr;
}
return {
// Suffix is a cache-space marker: embeddings.ts detects a name change and
// auto-invalidates any cached vectors built in a different space. '@graph'
// marked the switch off JS-side pooling; '-mcprefix' marks the #483 switch to
// the model-card role prefixes (old "query:"/"passage:" vectors are a
// different space and must be rebuilt).
name: "embedding-gemma@graph-mcprefix",
dim: DIM,
modelId: EMBEDDING_GEMMA_MODEL_ID,
embed: embedOne,
async embedBatch(texts) {
const out = [];
for (const t of texts) out.push(await embedOne(t));
return out;
}
};
}
// src/embedders/openai.ts
var OPENAI_3_LARGE_MODEL_ID = "text-embedding-3-large";
var OPENAI_3_LARGE_DIM = 3072;
var OPENAI_3_LARGE_MAX_INPUT_TOKENS = 8191;
var ENDPOINT = "https://api.openai.com/v1/embeddings";
var DEFAULT_MAX_BATCH_SIZE = 2048;
var DEFAULT_MAX_TOKENS_PER_REQUEST = 3e5;
var DEFAULT_TIMEOUT_MS = 3e4;
var DEFAULT_MAX_RETRIES = 3;
var DEFAULT_RETRY_BASE_MS = 1e3;
var MAX_RETRY_AFTER_MS = 6e4;
var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 503]);
function estimateTokens(text) {
return Math.ceil(text.length / 4);
}
function parseRetryAfterMs(header) {
if (!header) return null;
const secs = Number(header);
if (Number.isFinite(secs) && secs >= 0) {
return Math.min(secs * 1e3, MAX_RETRY_AFTER_MS);
}
const dateMs = Date.parse(header);
if (!Number.isNaN(dateMs)) {
return Math.min(Math.max(dateMs - Date.now(), 0), MAX_RETRY_AFTER_MS);
}
return null;
}
function readApiKey() {
const key = process.env.OPENAI_API_KEY;
if (!key || key.trim() === "") {
throw new Error(
"openai-3-large embedder requires OPENAI_API_KEY. Set the env var (export OPENAI_API_KEY=sk-...) or unset PLUR_EMBEDDER to fall back to the local default."
);
}
return key;
}
function clampToTokenLimit(text, index) {
const maxChars = OPENAI_3_LARGE_MAX_INPUT_TOKENS * 4;
if (text.length <= maxChars) return text;
logger.warning(
`[openai-3-large] input ${index} is ~${estimateTokens(text)} tokens (limit ${OPENAI_3_LARGE_MAX_INPUT_TOKENS}) \u2014 truncating to avoid an HTTP 400 mid-batch`
);
return text.slice(0, maxChars);
}
function parseVectors(json, expected) {
const data = json.data;
if (!Array.isArray(data) || data.length !== expected) {
throw new Error(
`openai-3-large embed returned ${data?.length ?? "no"} vectors for ${expected} inputs`
);
}
return data.map((row, i) => {
const e = row.embedding;
if (!Array.isArray(e)) {
throw new Error(`openai-3-large returned non-array embedding at index ${i}`);
}
if (e.length !== OPENAI_3_LARGE_DIM) {
throw new Error(
`openai-3-large returned ${e.length}-dim vector at index ${i}, expected ${OPENAI_3_LARGE_DIM}`
);
}
return Float32Array.from(e);
});
}
async function postEmbed(texts, opts) {
const key = readApiKey();
for (let attempt = 0; ; attempt++) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
let res;
try {
res = await opts.fetch(ENDPOINT, {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${key}`
},
body: JSON.stringify({
model: OPENAI_3_LARGE_MODEL_ID,
input: texts
}),
signal: controller.signal
});
} catch (err) {
if (controller.signal.aborted) {
throw new Error(`openai-3-large embed timed out after ${opts.timeoutMs}ms`);
}
throw err;
} finally {
clearTimeout(timer);
}
if (res.ok) {
return parseVectors(await res.json(), texts.length);
}
const body = await res.text().catch(() => "");
if (RETRYABLE_STATUS.has(res.status) && attempt < opts.maxRetries) {
const delay = parseRetryAfterMs(res.headers.get("retry-after")) ?? opts.retryBaseMs * 2 ** attempt;
logger.warning(
`[openai-3-large] HTTP ${res.status} \u2014 retrying in ${delay}ms (attempt ${attempt + 1}/${opts.maxRetries})`
);
await new Promise((r) => setTimeout(r, delay));
continue;
}
throw new Error(
`openai-3-large embed failed: HTTP ${res.status} ${res.statusText}${body ? ` \u2014 ${body.slice(0, 200)}` : ""}`
);
}
}
function chunkInputs(texts, opts) {
const chunks = [];
let current = [];
let currentTokens = 0;
for (const text of texts) {
const cost = estimateTokens(text);
if (current.length > 0 && (current.length >= opts.maxBatchSize || currentTokens + cost > opts.maxTokensPerRequest)) {
chunks.push(current);
current = [];
currentTokens = 0;
}
current.push(text);
currentTokens += cost;
}
if (current.length > 0) chunks.push(current);
return chunks;
}
function makeOpenAI3LargeAdapter(options) {
const opts = {
fetch: options?.fetch ?? ((...args) => globalThis.fetch(...args)),
timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
maxRetries: options?.maxRetries ?? DEFAULT_MAX_RETRIES,
retryBaseMs: options?.retryBaseMs ?? DEFAULT_RETRY_BASE_MS,
maxBatchSize: options?.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE,
maxTokensPerRequest: options?.maxTokensPerRequest ?? DEFAULT_MAX_TOKENS_PER_REQUEST
};
return {
name: "openai-3-large",
dim: OPENAI_3_LARGE_DIM,
modelId: OPENAI_3_LARGE_MODEL_ID,
async embed(text) {
const [v] = await postEmbed([clampToTokenLimit(text, 0)], opts);
return v;
},
async embedBatch(texts) {
if (texts.length === 0) return [];
const clamped = texts.map((t, i) => clampToTokenLimit(t, i));
const out = [];
for (const chunk of chunkInputs(clamped, opts)) {
out.push(...await postEmbed(chunk, opts));
}
return out;
}
};
}
// src/embedders/index.ts
var EMBEDDER_NAMES = [
"minilm",
"bge-small",
"bge-base",
"embedding-gemma",
"openai-3-large"
];
var DEFAULT_EMBEDDER = "bge-small";
var adapterCache = /* @__PURE__ */ new Map();
function _resetEmbedderCache() {
adapterCache.clear();
}
function getEmbedder(name) {
if (!EMBEDDER_NAMES.includes(name)) {
throw new Error(`Unknown embedder "${name}". Known: ${EMBEDDER_NAMES.join(", ")}`);
}
let adapter = adapterCache.get(name);
if (!adapter) {
adapter = build(name);
adapterCache.set(name, adapter);
}
return adapter;
}
function build(name) {
switch (name) {
case "minilm":
return makeMiniLMAdapter();
case "bge-small":
return makeBgeSmallAdapter();
case "bge-base":
return makeBgeBaseAdapter();
case "embedding-gemma":
return makeEmbeddingGemmaAdapter();
case "openai-3-large":
return makeOpenAI3LargeAdapter();
default: {
const _exhaustive = name;
throw new Error(`Unhandled embedder name: ${String(_exhaustive)}`);
}
}
}
var warnedUnknown = false;
function resolveEmbedderName(env = process.env) {
const raw = env.PLUR_EMBEDDER?.trim();
if (!raw) return DEFAULT_EMBEDDER;
if (EMBEDDER_NAMES.includes(raw)) return raw;
if (!warnedUnknown) {
logger.warning(
`[embedders] PLUR_EMBEDDER="${raw}" not recognised. Falling back to "${DEFAULT_EMBEDDER}". Known: ${EMBEDDER_NAMES.join(", ")}`
);
warnedUnknown = true;
}
return DEFAULT_EMBEDDER;
}
function _resetResolveWarnings() {
warnedUnknown = false;
}
export {
EMBEDDER_NAMES,
DEFAULT_EMBEDDER,
_resetEmbedderCache,
getEmbedder,
resolveEmbedderName,
_resetResolveWarnings
};
import {
atomicWrite
} from "./chunk-TXHLQGN3.js";
import {
engramSearchText
} from "./chunk-2UD5K4YD.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/embeddings.ts
import { existsSync, readFileSync, mkdirSync } from "fs";
import { join, dirname } from "path";
import { createHash } from "crypto";
var EMBED_DIM = 384;
var embedPipeline = null;
var lastLoadError = null;
var transformersUnavailable = false;
function readDisabledFromEnv(env) {
const raw = env.PLUR_DISABLE_EMBEDDINGS;
if (!raw) return null;
const normalized = raw.trim().toLowerCase();
if (normalized === "1" || normalized === "true" || normalized === "yes") {
return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var";
}
return null;
}
var ENV_DISABLED_REASON = readDisabledFromEnv(process.env);
var embeddingsDisabled = ENV_DISABLED_REASON !== null;
var disabledReason = ENV_DISABLED_REASON;
function embedderStatus() {
return {
available: !embeddingsDisabled && !transformersUnavailable,
loaded: embedPipeline !== null,
lastError: lastLoadError,
disabled: embeddingsDisabled,
disabledReason
};
}
function setEmbeddingsEnabled(enabled, reason) {
embeddingsDisabled = !enabled;
disabledReason = enabled ? null : reason ?? "embeddings disabled by config";
if (!enabled) {
embedPipeline = null;
}
}
function resetEmbedder() {
transformersUnavailable = false;
lastLoadError = null;
embedPipeline = null;
}
function _setCachedEmbedder(adapter) {
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
}
async function getEmbedder() {
if (embeddingsDisabled) return null;
if (embedPipeline) return embedPipeline;
try {
const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-PJW3I32N.js");
const adapter = getAdapter(resolveEmbedderName());
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
return embedPipeline;
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
return null;
}
}
async function embed(text, role) {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.embed === "function") {
let vector;
try {
vector = await embedder.embed(text, role);
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
embedPipeline = null;
return null;
}
if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) {
throw new Error(
`Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.`
);
}
return vector;
}
const result = await embedder(text, { pooling: "cls", normalize: true });
return new Float32Array(result.data);
}
async function getActiveEmbedderMeta() {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.name === "string" && typeof embedder.dim === "number") {
return { name: embedder.name, dim: embedder.dim };
}
return { name: "legacy-pipeline", dim: 0 };
}
async function activeEmbedderDim() {
const meta = await getActiveEmbedderMeta();
return meta && meta.dim > 0 ? meta.dim : null;
}
function cosineSimilarity(a, b) {
let dot = 0;
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
return dot;
}
var CACHE_VERSION = 1;
function emptyCache(meta) {
return {
meta: {
embedder_name: meta.name,
embedder_dim: meta.dim,
version: CACHE_VERSION
},
entries: {}
};
}
function loadCache(cachePath, active) {
if (!existsSync(cachePath)) return emptyCache(active);
try {
const raw = JSON.parse(readFileSync(cachePath, "utf8"));
if (!raw || typeof raw !== "object" || !raw.meta) {
logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`);
return emptyCache(active);
}
const meta = raw.meta;
if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) {
logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`);
return emptyCache(active);
}
const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {};
return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries };
} catch {
return emptyCache(active);
}
}
function saveCache(cachePath, cache) {
const dir = dirname(cachePath);
if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true });
atomicWrite(cachePath, JSON.stringify(cache), { durable: false });
}
function hashStatement(statement) {
return createHash("sha256").update(statement).digest("hex").slice(0, 16);
}
async function embeddingSearch(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const score = cosineSimilarity(queryEmbedding, engramEmbedding);
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit).map((s) => s.engram);
}
async function embeddingSearchWithScores(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding);
const score = Math.max(0, Math.min(1, rawScore));
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit);
}
async function rebuildJsonCache(engrams, storagePath, opts) {
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) {
return { reembedded: 0, skipped: true, reason: "embedder unavailable" };
}
const cachePath = join(storagePath, ".embeddings-cache.json");
const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta);
let count = 0;
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue;
const vec = await embed(searchText);
if (!vec) {
return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" };
}
cache.entries[engram.id] = { hash, embedding: Array.from(vec) };
count++;
}
saveCache(cachePath, cache);
return { reembedded: count, skipped: false };
}
export {
EMBED_DIM,
readDisabledFromEnv,
embedderStatus,
setEmbeddingsEnabled,
resetEmbedder,
_setCachedEmbedder,
embed,
activeEmbedderDim,
cosineSimilarity,
embeddingSearch,
embeddingSearchWithScores,
rebuildJsonCache
};
import {
DEFAULT_EMBEDDER,
EMBEDDER_NAMES,
_resetEmbedderCache,
_resetResolveWarnings,
getEmbedder,
resolveEmbedderName
} from "./chunk-VNFYIPQL.js";
import "./chunk-E4YVUWMJ.js";
export {
DEFAULT_EMBEDDER,
EMBEDDER_NAMES,
_resetEmbedderCache,
_resetResolveWarnings,
getEmbedder,
resolveEmbedderName
};
import {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
} from "./chunk-W55ICNED.js";
import "./chunk-TXHLQGN3.js";
import "./chunk-2UD5K4YD.js";
import "./chunk-E4YVUWMJ.js";
export {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
};
import {
MAX_SPACELESS_RUN_CHARS,
MIN_TOKEN_LENGTH,
TOKENIZER_VERSION,
computeIdf,
embeddingContentHash,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
hashEmbeddedText,
searchEngrams,
searchTextFrom,
termMatches
} from "./chunk-2UD5K4YD.js";
export {
MAX_SPACELESS_RUN_CHARS,
MIN_TOKEN_LENGTH,
TOKENIZER_VERSION,
computeIdf,
embeddingContentHash,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
hashEmbeddedText,
searchEngrams,
searchTextFrom,
termMatches
};
import {
appendHistory,
buildDedupPrompt,
computeContentHash,
maybeDailyBackup,
parseDedupResponse
} from "./chunk-EJQ4MGQV.js";
import {
withAsyncLock
} from "./chunk-TXHLQGN3.js";
import {
searchTextFrom
} from "./chunk-2UD5K4YD.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/learn-async.ts
var NEAR_DUPLICATE_OBSERVATION_FLOOR = 0.75;
async function persistOne(deps, corpus, changed) {
if (deps.store.updateMany) {
await deps.store.updateMany([changed]);
deps.store.invalidate();
return;
}
await deps.store.save(corpus);
}
async function withStoreLock(deps, fn) {
const guarded = async () => {
try {
maybeDailyBackup(deps.rootPath, deps.engramsPath);
} catch {
}
return await fn();
};
if (deps.store.withExclusiveAccess) return await deps.store.withExclusiveAccess(guarded);
return await withAsyncLock(deps.engramsPath, guarded);
}
function demoteIfSensitive(deps, engram, newStatement) {
const tags = Array.isArray(engram.tags) ? engram.tags.filter((t) => typeof t === "string") : [];
const scanText = tags.length ? `${newStatement}
${tags.join(" ")}` : newStatement;
const offending = deps.offendingHitsForScope(scanText, engram.scope ?? "global");
if (offending.length === 0) return;
const patterns = [...new Set(offending.map((h) => h.pattern))].join(", ");
logger.warning(
`[plur] sensitive content (${patterns}) held back from shared scope "${engram.scope}" \u2014 demoted to local/private so it is not written to a shared store. Re-scope deliberately if this is a false positive.`
);
const from = engram.scope ?? "global";
engram.scope = "local";
engram.visibility = "private";
engram.structured_data = {
...engram.structured_data ?? {},
_demoted: { from, to: "local", patterns }
};
}
async function executeDedupDecision(deps, statement, context, decision, targetId) {
switch (decision) {
case "NOOP": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing) return { engram: existing, decision: "NOOP", existing_id: targetId };
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "UPDATE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const updated = { ...engrams[idx] };
updated.statement = statement;
updated.content_hash = computeContentHash(statement);
updated.engram_version = (updated.engram_version ?? 1) + 1;
updated.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) updated.tags = [.../* @__PURE__ */ new Set([...updated.tags, ...context.tags])];
demoteIfSensitive(deps, updated, updated.statement);
engrams[idx] = updated;
await persistOne(deps, engrams, updated);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_updated",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { old_statement: existing.statement, new_statement: statement, reason: "LLM dedup UPDATE" }
});
return { engram: updated, decision: "UPDATE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "MERGE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const merged = { ...engrams[idx] };
merged.statement = `${merged.statement} ${statement}`;
merged.content_hash = computeContentHash(merged.statement);
merged.engram_version = (merged.engram_version ?? 1) + 1;
merged.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) merged.tags = [.../* @__PURE__ */ new Set([...merged.tags, ...context.tags])];
if (0.7 > merged.activation.retrieval_strength) merged.activation.retrieval_strength = 0.7;
demoteIfSensitive(deps, merged, merged.statement);
engrams[idx] = merged;
await persistOne(deps, engrams, merged);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_merged",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { merged_statement: statement, reason: "LLM dedup MERGE" }
});
return { engram: merged, decision: "MERGE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "ADD":
default:
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
}
async function learnAsync(deps, statement, context) {
const hashMatch = await deps.hashDedup(statement, context?.scope);
if (hashMatch) {
return { engram: hashMatch, decision: "NOOP", existing_id: hashMatch.id };
}
const { enabled = true, threshold = 0.85, mode = "llm" } = deps.dedupConfig;
if (!enabled || mode === "off") {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
let candidates = [];
try {
candidates = await deps.recallHybrid(statement, { limit: 5 });
} catch {
candidates = await deps.recall(statement, { limit: 5 });
}
if (candidates.length === 0) {
candidates = await deps.recall(statement, { limit: 5 });
}
candidates = candidates.filter((c) => c.status === "active");
if (context?.scope) {
candidates = candidates.filter((c) => c.scope === context.scope);
}
if (candidates.length === 0) {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
const llm = context?.llm;
let decision = "ADD";
let targetId = null;
let dedupMode = "hash-only";
let nearDuplicates;
if (mode === "llm" && llm && deps.isLlmAvailable()) {
dedupMode = "llm";
try {
const prompt = buildDedupPrompt(
statement,
candidates.map((c) => ({ id: c.id, statement: c.statement, type: c.type, domain: c.domain }))
);
const response = await llm(prompt);
const parsed = parseDedupResponse(response);
decision = parsed.decision;
targetId = parsed.target_id;
deps.recordLlmSuccess();
} catch (err) {
logger.warning(`LLM dedup failed, falling back to local similarity: ${err}`);
deps.recordLlmFailure();
decision = "ADD";
dedupMode = "hash-only";
}
}
if (dedupMode !== "llm" && deps.similarityScores) {
try {
const query = searchTextFrom({
statement,
domain: context?.domain,
tags: context?.tags,
rationale: context?.rationale,
source: context?.source,
dual_coding: context?.dual_coding,
knowledge_anchors: context?.knowledge_anchors
});
const scores = (await deps.similarityScores(query, candidates)).slice().sort((a, b) => b.score - a.score);
if (scores.length > 0) {
dedupMode = "cosine";
nearDuplicates = scores.slice(0, 3);
const top = scores[0];
if (top.score >= NEAR_DUPLICATE_OBSERVATION_FLOOR) {
try {
appendHistory(deps.rootPath, {
event: "dedup_near_duplicate",
engram_id: top.id,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: {
statement: statement.slice(0, 200),
top_score: Number(top.score.toFixed(4)),
reporting_threshold: threshold,
above_reporting_threshold: top.score >= threshold,
scope: context?.scope ?? null,
// The asymmetry that broke the bar. Recorded per write so the
// distribution can be split by it instead of averaging over it.
incoming_has_domain: Boolean(context?.domain),
incoming_has_rationale: Boolean(context?.rationale)
}
});
} catch (err) {
logger.warning(`could not record near-duplicate observation: ${err}`);
}
}
}
} catch (err) {
logger.warning(`local similarity dedup unavailable, adding without it: ${err}`);
dedupMode = "hash-only";
nearDuplicates = void 0;
}
}
const executed = await executeDedupDecision(deps, statement, context, decision, targetId);
return { ...executed, dedup: { mode: dedupMode, ...nearDuplicates ? { near_duplicates: nearDuplicates } : {} } };
}
async function learnBatch(deps, statements, llm, opts = {}) {
const results = [];
const failures = [];
const stats = { added: 0, updated: 0, merged: 0, noops: 0, failed: 0 };
const maxLlmCalls = opts.maxLlmCalls ?? 50;
let llmCallsUsed = 0;
let capWarned = false;
for (let i = 0; i < statements.length; i++) {
const { statement, context } = statements[i];
const stmtLlm = context?.llm ?? llm;
let effectiveLlm = stmtLlm;
if (stmtLlm) {
if (llmCallsUsed >= maxLlmCalls) {
effectiveLlm = void 0;
if (!capWarned) {
logger.warning(`learnBatch: maxLlmCalls (${maxLlmCalls}) reached \u2014 remaining statements fall back to local cosine dedup (see each result's dedup.mode)`);
capWarned = true;
}
} else {
effectiveLlm = async (prompt) => {
llmCallsUsed++;
return stmtLlm(prompt);
};
}
}
const ctx = { ...context, llm: effectiveLlm };
try {
const result = await learnAsync(deps, statement, ctx);
results.push({ ...result, input_index: i });
const key = result.decision.toLowerCase();
if (key === "noop") stats.noops++;
else if (key === "update") stats.updated++;
else if (key === "merge") stats.merged++;
else stats.added++;
} catch (err) {
stats.failed++;
failures.push({ index: i, statement, error: err instanceof Error ? err.message : String(err) });
logger.warning(`learnBatch: statement ${i} failed \u2014 ${err instanceof Error ? err.message : String(err)}`);
}
}
return { results, stats, failures };
}
export {
NEAR_DUPLICATE_OBSERVATION_FLOOR,
learnAsync,
learnBatch
};
+7
-4
{
"name": "@plur-ai/core",
"version": "0.17.2",
"version": "0.18.0",
"type": "module",

@@ -12,3 +12,3 @@ "main": "dist/index.js",

"@electric-sql/pglite": "^0.4.6",
"js-yaml": "^4.3.0",
"js-yaml": "^4.3.1",
"zod": "^3.23.0"

@@ -18,7 +18,7 @@ },

"@huggingface/transformers": "^3.8.1",
"better-sqlite3": "^11.0.0",
"better-sqlite3": "^12.0.0",
"pg": "^8.16.3"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.0",
"@types/better-sqlite3": "^9.0.0",
"@types/js-yaml": "^4.0.0",

@@ -30,2 +30,5 @@ "@types/pg": "^8.15.6",

},
"engines": {
"node": ">=20"
},
"license": "Apache-2.0",

@@ -32,0 +35,0 @@ "description": "Shared memory engine for AI agents — learn, recall, forget, feedback",

// src/fts.ts
import { createHash } from "crypto";
var STOP_WORDS = /* @__PURE__ */ new Set([
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"are",
"was",
"were",
"been",
"have",
"has",
"not",
"but",
"its",
"you",
"your",
"can",
"will",
"should",
"would",
"could",
"may",
"might"
]);
var MIN_TOKEN_LENGTH = 2;
var TOKENIZER_VERSION = 2;
function ftsTokenize(text) {
const lower = text.toLowerCase();
const tokens = lower.replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
for (const run of lower.match(new RegExp("\\p{Script=Han}{2,}", "gu")) ?? []) {
for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
}
return tokens;
}
function engramSearchText(engram) {
const parts = [engram.statement];
if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
if (engram.entities) {
for (const e of engram.entities) {
parts.push(e.name);
if (e.type !== "other") parts.push(e.type);
}
}
if (engram.temporal) {
if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
}
if (engram.rationale) parts.push(engram.rationale);
if (engram.source) parts.push(engram.source);
if (engram.dual_coding) {
if (engram.dual_coding.example) parts.push(engram.dual_coding.example);
if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy);
}
if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) {
for (const a of engram.knowledge_anchors) {
if (a.snippet) parts.push(a.snippet);
}
}
return parts.join(" ");
}
function embeddingContentHash(engram) {
return hashEmbeddedText(engramSearchText(engram));
}
function hashEmbeddedText(text) {
return createHash("md5").update(text).digest("hex");
}
function termMatches(t, qt) {
return t.includes(qt) || qt.startsWith(t);
}
function computeIdf(engrams, queryTokens, stats) {
if (stats) {
if (stats.N === 0) return /* @__PURE__ */ new Map();
const idf2 = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
const df = stats.df.get(qt) ?? 0;
idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df))));
}
return idf2;
}
const N = engrams.length;
if (N === 0) return /* @__PURE__ */ new Map();
const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
const idf = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
let df = 0;
for (const termSet of engramTermSets) {
if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) {
df++;
}
}
idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
}
return idf;
}
function extendCorpusStats(stats, queryTokens, outsiders) {
if (outsiders.length === 0) return stats;
const termSets = [];
let totalLen = 0;
for (const e of outsiders) {
const terms = ftsTokenize(engramSearchText(e));
totalLen += terms.length;
termSets.push(new Set(terms));
}
const df = new Map(stats.df);
for (const qt of queryTokens) {
let added = 0;
for (const set of termSets) {
if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++;
}
if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added);
}
const N = stats.N + outsiders.length;
return {
N,
df,
avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0
};
}
var BM25_K1 = 1.2;
var BM25_B = 0.75;
function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
const allTerms = ftsTokenize(engramSearchText(engram));
if (queryTokens.length === 0) return 0;
const docLen = allTerms.length;
const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
let score = 0;
for (const qt of queryTokens) {
let effectiveIdf;
if (!idfWeights) {
effectiveIdf = 1;
} else if (hasNonZeroIdf) {
effectiveIdf = idfWeights.get(qt) ?? 0;
if (effectiveIdf === 0) continue;
} else {
effectiveIdf = 1;
}
let tf = 0;
for (const t of allTerms) {
if (termMatches(t, qt)) tf++;
}
if (tf === 0) continue;
const numerator = tf * (BM25_K1 + 1);
const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
score += effectiveIdf * (numerator / denominator);
}
return score;
}
function searchEngrams(engrams, query, limit = 20, stats) {
const queryTokens = ftsTokenize(query);
if (queryTokens.length === 0) return [];
const idfWeights = computeIdf(engrams, queryTokens, stats);
const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0);
if (scored.length === 0) {
scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0);
}
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
}
export {
MIN_TOKEN_LENGTH,
TOKENIZER_VERSION,
ftsTokenize,
engramSearchText,
embeddingContentHash,
hashEmbeddedText,
termMatches,
computeIdf,
extendCorpusStats,
ftsScore,
searchEngrams
};
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/embedders/transformers-base.ts
var pipelineCache = /* @__PURE__ */ new Map();
async function loadPipeline(modelId, dtype) {
const key = `${modelId}::${dtype ?? "fp32"}`;
let pending = pipelineCache.get(key);
if (!pending) {
pending = (async () => {
process.env.HF_HUB_DISABLE_XET ??= "1";
const { pipeline } = await import("@huggingface/transformers");
return pipeline("feature-extraction", modelId, dtype ? { dtype } : void 0);
})();
pipelineCache.set(key, pending);
}
return await pending;
}
function makeTransformersAdapter(config) {
const pooling = config.pooling;
const normalize = config.normalize ?? true;
async function embedOne(text) {
const pipe = await loadPipeline(config.modelId, config.dtype);
const result = await pipe(text, { pooling, normalize });
const arr = result.data instanceof Float32Array ? result.data : new Float32Array(result.data);
if (arr.length !== config.dim) {
throw new Error(
`Embedder "${config.name}" returned ${arr.length}-dim vector, expected ${config.dim}`
);
}
return arr;
}
return {
name: config.name,
dim: config.dim,
modelId: config.modelId,
embed: embedOne,
async embedBatch(texts) {
const out = [];
for (const t of texts) out.push(await embedOne(t));
return out;
}
};
}
// src/embedders/minilm.ts
var MINILM_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
function makeMiniLMAdapter() {
return makeTransformersAdapter({
name: "minilm",
dim: 384,
modelId: MINILM_MODEL_ID,
pooling: "mean",
normalize: true,
dtype: "fp32"
});
}
// src/embedders/bge-small.ts
var BGE_SMALL_MODEL_ID = "Xenova/bge-small-en-v1.5";
function makeBgeSmallAdapter() {
return makeTransformersAdapter({
name: "bge-small",
dim: 384,
modelId: BGE_SMALL_MODEL_ID,
pooling: "cls",
normalize: true,
dtype: "fp32"
});
}
// src/embedders/bge-base.ts
var BGE_BASE_MODEL_ID = "Xenova/bge-base-en-v1.5";
function makeBgeBaseAdapter() {
return makeTransformersAdapter({
name: "bge-base",
dim: 768,
modelId: BGE_BASE_MODEL_ID,
pooling: "cls",
normalize: true,
dtype: "fp32"
});
}
// src/embedders/embedding-gemma.ts
var EMBEDDING_GEMMA_MODEL_ID = "onnx-community/embeddinggemma-300m-ONNX";
var DIM = 768;
var loaded = null;
async function load() {
if (!loaded) {
loaded = (async () => {
process.env.HF_HUB_DISABLE_XET ??= "1";
const { AutoTokenizer, AutoModel } = await import("@huggingface/transformers");
const tokenizer = await AutoTokenizer.from_pretrained(EMBEDDING_GEMMA_MODEL_ID);
const model = await AutoModel.from_pretrained(EMBEDDING_GEMMA_MODEL_ID, { dtype: "q8" });
return { tokenizer, model };
})();
}
return loaded;
}
function makeEmbeddingGemmaAdapter() {
async function embedOne(text, role) {
const prefix = role === "query" ? "task: search result | query: " : "title: none | text: ";
const { tokenizer, model } = await load();
const inputs = await tokenizer(prefix + text, { padding: true, truncation: true });
const outputs = await model(inputs);
const raw = outputs.sentence_embedding.data;
const arr = raw instanceof Float32Array ? raw : new Float32Array(raw);
if (arr.length !== DIM) {
throw new Error(`EmbeddingGemma: expected ${DIM}-dim sentence_embedding, got ${arr.length}`);
}
return arr;
}
return {
// Suffix is a cache-space marker: embeddings.ts detects a name change and
// auto-invalidates any cached vectors built in a different space. '@graph'
// marked the switch off JS-side pooling; '-mcprefix' marks the #483 switch to
// the model-card role prefixes (old "query:"/"passage:" vectors are a
// different space and must be rebuilt).
name: "embedding-gemma@graph-mcprefix",
dim: DIM,
modelId: EMBEDDING_GEMMA_MODEL_ID,
embed: embedOne,
async embedBatch(texts) {
const out = [];
for (const t of texts) out.push(await embedOne(t));
return out;
}
};
}
// src/embedders/openai.ts
var OPENAI_3_LARGE_MODEL_ID = "text-embedding-3-large";
var OPENAI_3_LARGE_DIM = 3072;
var OPENAI_3_LARGE_MAX_INPUT_TOKENS = 8191;
var ENDPOINT = "https://api.openai.com/v1/embeddings";
var DEFAULT_MAX_BATCH_SIZE = 2048;
var DEFAULT_MAX_TOKENS_PER_REQUEST = 3e5;
var DEFAULT_TIMEOUT_MS = 3e4;
var DEFAULT_MAX_RETRIES = 3;
var DEFAULT_RETRY_BASE_MS = 1e3;
var MAX_RETRY_AFTER_MS = 6e4;
var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 503]);
function estimateTokens(text) {
return Math.ceil(text.length / 4);
}
function parseRetryAfterMs(header) {
if (!header) return null;
const secs = Number(header);
if (Number.isFinite(secs) && secs >= 0) {
return Math.min(secs * 1e3, MAX_RETRY_AFTER_MS);
}
const dateMs = Date.parse(header);
if (!Number.isNaN(dateMs)) {
return Math.min(Math.max(dateMs - Date.now(), 0), MAX_RETRY_AFTER_MS);
}
return null;
}
function readApiKey() {
const key = process.env.OPENAI_API_KEY;
if (!key || key.trim() === "") {
throw new Error(
"openai-3-large embedder requires OPENAI_API_KEY. Set the env var (export OPENAI_API_KEY=sk-...) or unset PLUR_EMBEDDER to fall back to the local default."
);
}
return key;
}
function clampToTokenLimit(text, index) {
const maxChars = OPENAI_3_LARGE_MAX_INPUT_TOKENS * 4;
if (text.length <= maxChars) return text;
logger.warning(
`[openai-3-large] input ${index} is ~${estimateTokens(text)} tokens (limit ${OPENAI_3_LARGE_MAX_INPUT_TOKENS}) \u2014 truncating to avoid an HTTP 400 mid-batch`
);
return text.slice(0, maxChars);
}
function parseVectors(json, expected) {
const data = json.data;
if (!Array.isArray(data) || data.length !== expected) {
throw new Error(
`openai-3-large embed returned ${data?.length ?? "no"} vectors for ${expected} inputs`
);
}
return data.map((row, i) => {
const e = row.embedding;
if (!Array.isArray(e)) {
throw new Error(`openai-3-large returned non-array embedding at index ${i}`);
}
if (e.length !== OPENAI_3_LARGE_DIM) {
throw new Error(
`openai-3-large returned ${e.length}-dim vector at index ${i}, expected ${OPENAI_3_LARGE_DIM}`
);
}
return Float32Array.from(e);
});
}
async function postEmbed(texts, opts) {
const key = readApiKey();
for (let attempt = 0; ; attempt++) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
let res;
try {
res = await opts.fetch(ENDPOINT, {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${key}`
},
body: JSON.stringify({
model: OPENAI_3_LARGE_MODEL_ID,
input: texts
}),
signal: controller.signal
});
} catch (err) {
if (controller.signal.aborted) {
throw new Error(`openai-3-large embed timed out after ${opts.timeoutMs}ms`);
}
throw err;
} finally {
clearTimeout(timer);
}
if (res.ok) {
return parseVectors(await res.json(), texts.length);
}
const body = await res.text().catch(() => "");
if (RETRYABLE_STATUS.has(res.status) && attempt < opts.maxRetries) {
const delay = parseRetryAfterMs(res.headers.get("retry-after")) ?? opts.retryBaseMs * 2 ** attempt;
logger.warning(
`[openai-3-large] HTTP ${res.status} \u2014 retrying in ${delay}ms (attempt ${attempt + 1}/${opts.maxRetries})`
);
await new Promise((r) => setTimeout(r, delay));
continue;
}
throw new Error(
`openai-3-large embed failed: HTTP ${res.status} ${res.statusText}${body ? ` \u2014 ${body.slice(0, 200)}` : ""}`
);
}
}
function chunkInputs(texts, opts) {
const chunks = [];
let current = [];
let currentTokens = 0;
for (const text of texts) {
const cost = estimateTokens(text);
if (current.length > 0 && (current.length >= opts.maxBatchSize || currentTokens + cost > opts.maxTokensPerRequest)) {
chunks.push(current);
current = [];
currentTokens = 0;
}
current.push(text);
currentTokens += cost;
}
if (current.length > 0) chunks.push(current);
return chunks;
}
function makeOpenAI3LargeAdapter(options) {
const opts = {
fetch: options?.fetch ?? ((...args) => globalThis.fetch(...args)),
timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
maxRetries: options?.maxRetries ?? DEFAULT_MAX_RETRIES,
retryBaseMs: options?.retryBaseMs ?? DEFAULT_RETRY_BASE_MS,
maxBatchSize: options?.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE,
maxTokensPerRequest: options?.maxTokensPerRequest ?? DEFAULT_MAX_TOKENS_PER_REQUEST
};
return {
name: "openai-3-large",
dim: OPENAI_3_LARGE_DIM,
modelId: OPENAI_3_LARGE_MODEL_ID,
async embed(text) {
const [v] = await postEmbed([clampToTokenLimit(text, 0)], opts);
return v;
},
async embedBatch(texts) {
if (texts.length === 0) return [];
const clamped = texts.map((t, i) => clampToTokenLimit(t, i));
const out = [];
for (const chunk of chunkInputs(clamped, opts)) {
out.push(...await postEmbed(chunk, opts));
}
return out;
}
};
}
// src/embedders/index.ts
var EMBEDDER_NAMES = [
"minilm",
"bge-small",
"bge-base",
"embedding-gemma",
"openai-3-large"
];
var DEFAULT_EMBEDDER = "bge-small";
var adapterCache = /* @__PURE__ */ new Map();
function _resetEmbedderCache() {
adapterCache.clear();
}
function getEmbedder(name) {
if (!EMBEDDER_NAMES.includes(name)) {
throw new Error(`Unknown embedder "${name}". Known: ${EMBEDDER_NAMES.join(", ")}`);
}
let adapter = adapterCache.get(name);
if (!adapter) {
adapter = build(name);
adapterCache.set(name, adapter);
}
return adapter;
}
function build(name) {
switch (name) {
case "minilm":
return makeMiniLMAdapter();
case "bge-small":
return makeBgeSmallAdapter();
case "bge-base":
return makeBgeBaseAdapter();
case "embedding-gemma":
return makeEmbeddingGemmaAdapter();
case "openai-3-large":
return makeOpenAI3LargeAdapter();
default: {
const _exhaustive = name;
throw new Error(`Unhandled embedder name: ${String(_exhaustive)}`);
}
}
}
var warnedUnknown = false;
function resolveEmbedderName(env = process.env) {
const raw = env.PLUR_EMBEDDER?.trim();
if (!raw) return DEFAULT_EMBEDDER;
if (EMBEDDER_NAMES.includes(raw)) return raw;
if (!warnedUnknown) {
logger.warning(
`[embedders] PLUR_EMBEDDER="${raw}" not recognised. Falling back to "${DEFAULT_EMBEDDER}". Known: ${EMBEDDER_NAMES.join(", ")}`
);
warnedUnknown = true;
}
return DEFAULT_EMBEDDER;
}
function _resetResolveWarnings() {
warnedUnknown = false;
}
export {
EMBEDDER_NAMES,
DEFAULT_EMBEDDER,
_resetEmbedderCache,
getEmbedder,
resolveEmbedderName,
_resetResolveWarnings
};
import {
atomicWrite,
withLock
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/schemas/engram.ts
import { z } from "zod";
var ActivationSchema = z.object({
retrieval_strength: z.number().min(0).max(1),
storage_strength: z.number().min(0).max(1),
frequency: z.number().int().min(0),
last_accessed: z.string().describe("Date or ISO 8601 timestamp of last access.")
}).describe("ACT-R activation parameters driving decay and ranking. STABLE.");
var KnowledgeTypeSchema = z.object({
memory_class: z.enum(["semantic", "episodic", "procedural", "metacognitive"]),
cognitive_level: z.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"]).describe("Bloom's taxonomy level.")
});
var KnowledgeAnchorSchema = z.object({
path: z.string().describe("Path to a grounding document/file."),
relevance: z.enum(["primary", "supporting", "example"]).default("supporting"),
snippet: z.string().max(200).optional(),
snippet_extracted_at: z.string().optional()
});
var AssociationSchema = z.object({
target_type: z.enum(["engram", "document"]),
target: z.string().describe("ID or path of the association target."),
strength: z.number().min(0).max(0.95),
type: z.enum(["semantic", "temporal", "causal", "co_accessed"]),
updated_at: z.string().optional()
});
var DualCodingSchema = z.object({
example: z.string().optional(),
analogy: z.string().optional()
}).describe("Worked example and/or analogy (dual coding). At least one of example or analogy MUST be provided (enforced at runtime by the Zod .refine below).").refine(
(d) => d.example || d.analogy,
"At least one of example or analogy must be provided"
);
var RelationsSchema = z.object({
broader: z.array(z.string()).default([]),
narrower: z.array(z.string()).default([]),
related: z.array(z.string()).default([]),
conflicts: z.array(z.string()).default([]),
/** IDs of engrams this one intentionally replaces (#240). An intentional
* update is not a tension — the scanner skips supersedes-linked pairs. */
supersedes: z.array(z.string()).default([]),
/** Reverse edge of `supersedes` (#240) — IDs of engrams that replace this one. */
superseded_by: z.array(z.string()).default([])
}).describe("Typed graph edges between engram IDs.");
var ProvenanceSchema = z.object({
origin: z.string(),
chain: z.array(z.string()).default([]),
signature: z.string().nullable().default(null).describe("RESERVED. Detached signature over the engram. Algorithm and canonicalization not yet specified \u2014 see ENGRAM-STANDARD-v1.md \xA77."),
license: z.string().default("cc-by-sa-4.0")
}).describe("Origin and signing chain. STABLE for origin/chain/license; signature is RESERVED (see ENGRAM-STANDARD-v1.md \xA77).");
var FeedbackSignalsSchema = z.object({
positive: z.number().int().default(0),
negative: z.number().int().default(0),
neutral: z.number().int().default(0)
});
var EntityRefSchema = z.object({
name: z.string(),
type: z.enum([
"person",
"organization",
"technology",
"concept",
"project",
"tool",
"place",
"event",
"standard",
"other"
]),
uri: z.string().url().optional()
});
var TemporalSchema = z.object({
learned_at: z.string(),
valid_from: z.string().optional(),
valid_until: z.string().optional(),
ingested_at: z.string().optional()
}).describe("Bi-temporal anchoring (Zep-inspired). When is this knowledge true?");
var UsageStatsSchema = z.object({
injections: z.number().int().default(0),
hits: z.number().int().default(0),
misses: z.number().int().default(0),
last_hit_at: z.string().optional()
});
var EpisodicFieldsSchema = z.object({
emotional_weight: z.number().int().min(1).max(10).default(5),
confidence: z.number().int().min(1).max(10).default(5),
trigger_context: z.string().optional(),
journal_ref: z.string().optional()
});
var PreviousVersionRefSchema = z.object({
event_id: z.string(),
changed_at: z.string()
});
var ExchangeMetadataSchema = z.object({
fitness_score: z.number().min(0).max(1).optional(),
environmental_diversity: z.number().int().default(0),
adoption_count: z.number().int().default(0),
contradiction_rate: z.number().min(0).max(1).default(0)
});
var SerendipitySchema = z.object({
unexpectedness: z.number().min(0).max(1),
relevance: z.number().min(0).max(1),
score: z.number().min(0).max(1)
});
var InsightFateSchema = z.enum([
"surfaced",
// shown in a briefing; no downstream action yet
"promoted",
// became a durable engram / zettel
"cited",
// referenced in later journal/work
"tasked",
// converted to a GTD task
"dismissed",
// user/LLM rejected it
"expired"
// decayed out of the buffer unused
]);
var InsightFieldSchema = z.object({
/** Which memory-stream operation produced this insight. Nightly arc:
* `distill` (episode→insight synthesis) → `consolidate` (convergent gist
* abstraction over the buffer) → `dream` (divergent REM-style recombination —
* speculative, never auto-promoted). `connect`/`emerge`/`drift` are on-demand lenses. */
operation: z.enum(["distill", "consolidate", "dream", "connect", "emerge", "drift"]),
synthesized_at: z.string(),
/** Anti-hallucination grounding. Cited source notes live in the parent engram's
* `knowledge_anchors[]`; this flags whether the claim was verified against those
* snippets. `ungrounded` = couldn't cite sources → quarantined (`candidate`, never
* surfaced). `speculative` = a `dream`: its recombined INPUTS are cited but its
* CONCLUSION is an explicit hypothesis — surfaced only as inspiration, and (per the
* promote-requires-grounding refine below) it must be re-grounded to `verified`
* before it can be promoted to a durable engram. */
grounding: z.enum(["verified", "unverified", "ungrounded", "speculative"]).default("unverified"),
/** The episode-log slice this insight was distilled from (evidence trail). */
source_episode_ids: z.array(z.string()).default([]),
/** Distinct objective for connect/emerge/dream insights. */
serendipity: SerendipitySchema.optional(),
fate: InsightFateSchema.default("surfaced"),
/** Engram id / zettel path / task id the insight became, if acted upon. */
fate_ref: z.string().optional(),
fate_at: z.string().optional(),
/** How many briefings have surfaced this insight (acted-upon-rate denominator). */
surfaced_count: z.number().int().min(0).default(0)
}).refine(
// Promote-requires-grounding (user rule 2026-06-15): a dream is inspiration, not
// fact. A speculative/ungrounded insight can only become a durable promotion once
// it has been re-grounded in reality (grounding=verified).
(i) => i.fate !== "promoted" || i.grounding === "verified",
{ message: "A promoted insight must be grounded (grounding=verified); speculative dreams cannot be promoted until re-grounded.", path: ["grounding"] }
);
var ExtractionProvenanceSchema = z.object({
confidence: z.number().min(0).max(1).optional().describe("0-1 classifier confidence at extraction time. Frozen at write; distinct from feedback-derived computeConfidence() and from episodic.confidence."),
source_commit: z.string().optional().describe("Git SHA of the source repository at extraction time (reproducibility)."),
extractor_version: z.string().optional().describe("Version of the extracting CLI/tool (schema-migration handle). Complementary to the pack-level capsule producer field (#61).")
}).passthrough().describe("ETL extraction provenance convention carried in structured_data.extraction (#463). Not wired into EngramSchema.");
function getExtractionProvenance(engram) {
const extraction = engram.structured_data?.["extraction"];
if (extraction === void 0 || extraction === null) return null;
const parsed = ExtractionProvenanceSchema.safeParse(extraction);
return parsed.success ? parsed.data : null;
}
var EngramSchema = z.object({
// Identity
id: z.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/).describe("Unique identifier. Class prefix ENG (concrete engram), ABS (abstraction), or META (meta-engram). Canonical concrete form: ENG-YYYY-MMDD-NNN; store-namespaced form: ENG-{PREFIX}-YYYY-MMDD-NNN."),
version: z.number().int().min(1).default(2).describe("Schema-shape generation of this engram object (currently 2). Distinct from engram_version, which tracks content evolution."),
// 'active' and 'retired' are the two states any current code path assigns
// (retire via forget/dedup/supersede). 'dormant' and 'candidate' are NOT
// assigned by any code today: 'dormant' was only ever set by the batchDecay
// pass removed in #563 (decay is now a read-time property, not a materialized
// status), and 'candidate' is reserved. They are kept in the enum so stores
// written before #563 that persisted status:'dormant' still load, and so the
// status filter accepts them; do not remove without a data migration.
status: z.enum(["active", "dormant", "retired", "candidate"]).describe("Lifecycle state. Assigned values today are active/retired; dormant/candidate are legacy/reserved (see note above)."),
consolidated: z.boolean().default(false).describe("Whether this engram has been through consolidation (sleep-like batch reprocessing)."),
type: z.enum(["behavioral", "terminological", "procedural", "architectural"]).describe("Top-level classification of the knowledge."),
scope: z.string().describe("Hierarchical namespace, e.g. 'global', 'project:my-app', 'group:plur/test'. Free-form string; ':' separates scope kind from path."),
visibility: z.enum(["private", "public", "template"]).default("private").describe("Sharing posture. 'private' engrams MUST NOT be exported in packs."),
// Content
statement: z.string().min(1).describe("The assertion itself \u2014 the load-bearing content of the engram."),
rationale: z.string().optional().describe("Why this is true / why it matters."),
contraindications: z.array(z.string()).optional().describe("Conditions under which the statement does NOT apply."),
// Lineage
source: z.string().optional().describe("Free-text origin (session, document, conversation)."),
source_patterns: z.array(z.string()).optional().describe("Pattern IDs that contributed to this engram."),
derivation_count: z.number().int().min(0).default(1).describe("How many derivation steps produced this engram."),
pack: z.string().nullable().default(null).describe("Name of the pack this engram belongs to, or null."),
abstract: z.string().nullable().default(null).describe("ID of an ABS- abstraction this engram instantiates, or null."),
derived_from: z.string().nullable().default(null).describe("ID of the engram this was derived from, or null."),
// Classification
knowledge_type: KnowledgeTypeSchema.optional(),
domain: z.string().optional().describe("Dotted domain path, e.g. 'dev/testing' or 'plur.session'."),
tags: z.array(z.string()).default([]).describe("Free-form tags used for matching and retrieval."),
// Activation (ACT-R model)
activation: ActivationSchema.default({
retrieval_strength: 0.7,
storage_strength: 1,
frequency: 0,
last_accessed: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
}),
// Relations & grounding
relations: RelationsSchema.optional(),
associations: z.array(AssociationSchema).default([]),
knowledge_anchors: z.array(KnowledgeAnchorSchema).default([]),
dual_coding: DualCodingSchema.optional(),
// Provenance
provenance: ProvenanceSchema.optional(),
// Feedback
feedback_signals: FeedbackSignalsSchema.default({ positive: 0, negative: 0, neutral: 0 }),
// === NEW OPTIONAL FIELDS (v2.1) ===
/** Typed entity references extracted from statement. Enables graph queries. */
entities: z.array(EntityRefSchema).optional().describe("Typed entity references extracted from statement. Enables graph queries."),
/** Temporal validity window. When is this knowledge true? */
temporal: TemporalSchema.optional(),
/** Automatic usage tracking. Injections, hits, misses. */
usage: UsageStatsSchema.optional(),
/** Episodic context: emotional weight, confidence, trigger. */
episodic: EpisodicFieldsSchema.optional(),
/** Exchange marketplace metadata: fitness, adoption, diversity. */
exchange: ExchangeMetadataSchema.optional(),
/** Extensible key-value data for domain-specific fields. */
structured_data: z.record(z.string(), z.unknown()).optional().describe("Extensible key-value data for domain-specific fields."),
/** Memory-stream insight provenance (metacognition Phase 1). Orthogonal to
* `type`. Present iff this engram was synthesized by the metacognition memory
* stream; the episodic insight buffer is the set of engrams where this is set. */
insight: InsightFieldSchema.optional(),
/** Polarity classification: 'do' for directives, 'dont' for prohibitions, null for unclassified. */
polarity: z.enum(["do", "dont"]).nullable().default(null).describe("'do' for directives, 'dont' for prohibitions, null for unclassified."),
// === SP1: Memory Intelligence fields ===
content_hash: z.string().optional().describe("Hash of normalized statement content, used for dedup."),
commitment: z.enum(["exploring", "leaning", "decided", "locked"]).optional().describe("Commitment level of the asserted knowledge."),
locked_at: z.string().optional().describe("Timestamp when commitment reached 'locked'."),
locked_reason: z.string().optional().describe("Why this engram was locked."),
// === SP1: Reference counting (issue #107) ===
/** Number of write attempts that resolved to this engram.
* Incremented on every hash-dedup hit; decremented by forget().
* Engram physically retires only when this reaches 0. */
reference_count: z.number().int().min(0).default(1).describe("Number of write attempts that resolved to this engram (same-scope re-learns). Engram retires only when this reaches 0."),
/** Provenance of each write attempt. One entry per write (including the
* first). Migrated old engrams without this field start with []. */
sources: z.array(z.object({
scope: z.string(),
session_id: z.string().nullable().default(null),
stored_at: z.string().describe("ISO 8601 timestamp of this write.")
})).default([]).describe("Provenance of each write attempt; one entry per write."),
// === SP1: Cross-scope recurrence (issue #176) ===
/** Number of times this engram's content was re-learned at a DIFFERENT
* scope than the original. Triggers auto-broadening + commitment
* escalation when threshold is crossed. Distinct from reference_count
* (which counts re-learns in the SAME scope) — recurrence_count is
* evidence of universal applicability, not just repetition. */
recurrence_count: z.number().int().min(0).default(0).describe("Number of times this content was re-learned at a DIFFERENT scope than the original. Evidence of universal applicability."),
// === SP2: History & Evolution fields ===
engram_version: z.number().int().min(1).default(1).describe("Content-evolution version (incremented when the statement materially changes)."),
previous_version_ref: PreviousVersionRefSchema.optional(),
episode_ids: z.array(z.string()).default([]).describe("IDs of episodes (raw conversational events) that produced or reinforced this engram."),
// === SP3: Retrieval & Injection fields ===
summary: z.string().max(80).optional().describe("Short (<=80 char) injection-friendly summary."),
/**
* Always-load flag. Pinned engrams bypass the term-hits gate in scoreEngram
* and are eligible for injection on every session start, regardless of
* keyword overlap with the user's task. Use sparingly: meta-rules,
* cross-cutting safety conventions, and core operating principles only.
* Pinned engrams still respect the token budget — they bypass per-pack and
* per-domain fairness caps in fillTokenBudget so always-load behavior is
* honored even if a single pack contributes many.
*/
pinned: z.boolean().optional().describe("Always-load flag. Pinned engrams bypass the keyword-relevance gate and are eligible for injection every session. Use sparingly.")
});
var EngramSchemaPassthrough = EngramSchema.passthrough();
// src/backup.ts
import * as fs from "fs";
import * as path from "path";
import { createHash } from "crypto";
import * as yaml from "js-yaml";
var BACKUP_DIR = "backups";
var KEEP_DAILY = 7;
var KEEP_WEEKLY = 4;
var SHRINK_TOLERANCE = 0.1;
function statePath(root) {
return path.join(root, BACKUP_DIR, ".state.json");
}
function readState(root) {
try {
return JSON.parse(fs.readFileSync(statePath(root), "utf8"));
} catch {
return {};
}
}
function writeState(root, state) {
const p = statePath(root);
fs.mkdirSync(path.dirname(p), { recursive: true });
fs.writeFileSync(p, JSON.stringify(state, null, 2) + "\n", "utf8");
}
function sha256(content) {
return createHash("sha256").update(content).digest("hex");
}
function validateStore(filePath, lastGoodCount) {
const failures = [];
const reasons = [];
let raw;
try {
raw = fs.readFileSync(filePath);
} catch (err) {
return { ok: false, failures: ["unreadable"], reasons: [`cannot read ${filePath}: ${err}`], count: null };
}
if (raw.length === 0) {
return { ok: false, failures: ["empty"], reasons: ["file is 0 bytes"], count: null };
}
if (!raw.toString("utf8").endsWith("\n")) {
failures.push("truncated");
reasons.push("file does not end with a newline \u2014 PLUR's writer always emits one, so this looks cut short");
}
let doc;
try {
doc = yaml.load(raw.toString("utf8"));
} catch (err) {
return { ok: false, failures: ["unparseable"], reasons: [`YAML parse failed: ${err}`], count: null };
}
if (doc == null || typeof doc !== "object" || Array.isArray(doc) || !Array.isArray(doc.engrams)) {
return {
ok: false,
failures: ["not-a-store"],
reasons: ["parsed, but is not a mapping with an `engrams` list"],
count: null
};
}
const entries = doc.engrams;
const count = entries.length;
let invalid = 0;
const ids = /* @__PURE__ */ new Set();
let duplicateIds = 0;
let missingIds = 0;
for (const entry of entries) {
if (!EngramSchemaPassthrough.safeParse(entry).success) invalid++;
const id = entry?.id;
if (typeof id !== "string" || id.length === 0) missingIds++;
else if (ids.has(id)) duplicateIds++;
else ids.add(id);
}
if (invalid > 0) {
failures.push("invalid-entries");
reasons.push(`${invalid} entry/entries fail schema validation`);
}
if (missingIds > 0) {
failures.push("missing-ids");
reasons.push(`${missingIds} entry/entries have no id`);
}
if (duplicateIds > 0) {
failures.push("duplicate-ids");
reasons.push(`${duplicateIds} duplicate id(s)`);
}
if (typeof lastGoodCount === "number" && lastGoodCount > 0) {
const floor = lastGoodCount * (1 - SHRINK_TOLERANCE);
if (count < floor) {
failures.push("shrunk");
reasons.push(
`holds ${count} engram(s) but the last good snapshot held ${lastGoodCount} \u2014 a drop this large is how a truncation looks`
);
}
}
return { ok: failures.length === 0, failures, reasons, count };
}
function todayStamp(now) {
return now.toISOString().slice(0, 10);
}
function snapshotPath(root, stamp) {
return path.join(root, BACKUP_DIR, `engrams-${stamp}.yaml`);
}
var doneThisProcess = /* @__PURE__ */ new Set();
function maybeDailyBackup(root, storePath, now = /* @__PURE__ */ new Date()) {
const key = `${root}\0${todayStamp(now)}`;
if (doneThisProcess.has(key)) return { taken: false, skipped: "already-today" };
try {
if (!fs.existsSync(storePath)) {
doneThisProcess.add(key);
return { taken: false, skipped: "no-store" };
}
const state = readState(root);
const stamp = todayStamp(now);
if (state.last_backup_date === stamp) {
doneThisProcess.add(key);
return { taken: false, skipped: "already-today" };
}
const existing = listBackups(root);
const strongest = existing.reduce(
(max, b) => typeof b.count === "number" && (max === void 0 || b.count > max) ? b.count : max,
void 0
);
const baseline = state.last_good_count ?? strongest;
const validity = validateStore(storePath, baseline);
if (!validity.ok) {
logger.warning(
`[plur:backup] refusing to snapshot ${storePath} \u2014 ${validity.reasons.join("; ")}. Your last good backup is unchanged. Run 'plur doctor' to inspect.`
);
return { taken: false, skipped: "invalid", validity };
}
const bytes = fs.readFileSync(storePath);
const dest = snapshotPath(root, stamp);
const sameDay = existing.find((b) => b.stamp === stamp);
if (sameDay && typeof sameDay.count === "number" && (validity.count ?? 0) < sameDay.count) {
logger.warning(
`[plur:backup] keeping today's existing snapshot (${sameDay.count} engrams) \u2014 the live store holds ${validity.count}, and replacing a stronger snapshot with a weaker one would discard the better copy. Run 'plur restore --list' to inspect.`
);
doneThisProcess.add(key);
return { taken: false, skipped: "invalid", validity };
}
fs.mkdirSync(path.dirname(dest), { recursive: true });
writeFileDurable(dest, bytes);
writeFileDurable(
`${dest}.sha256`,
Buffer.from(
`${sha256(bytes)} ${path.basename(dest)}
${validity.count} engrams
taken_at ${now.toISOString()}
`,
"utf8"
)
);
doneThisProcess.add(key);
writeState(root, {
last_backup_date: stamp,
last_good_count: validity.count ?? void 0,
last_good_sha256: sha256(bytes)
});
rotate(root, now);
return { taken: true, path: dest, validity };
} catch (err) {
logger.warning(`[plur:backup] snapshot failed (the write itself was unaffected): ${err}`);
return { taken: false, skipped: "invalid" };
}
}
function flushFileAt(filePath) {
let fd;
try {
fd = fs.openSync(filePath, "r+");
fs.fsyncSync(fd);
} catch {
} finally {
if (fd !== void 0) {
try {
fs.closeSync(fd);
} catch {
}
}
}
}
function writeFileDurable(dest, bytes) {
const fd = fs.openSync(dest, "w");
try {
fs.writeFileSync(fd, bytes);
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
}
function listBackups(root) {
const dir = path.join(root, BACKUP_DIR);
if (!fs.existsSync(dir)) return [];
const out = [];
for (const name of fs.readdirSync(dir)) {
const m = name.match(/^engrams-(\d{4}-\d{2}-\d{2})\.yaml$/);
if (!m) continue;
const full = path.join(dir, name);
const entry = { path: full, stamp: m[1], size: fs.statSync(full).size };
try {
const sidecar = fs.readFileSync(`${full}.sha256`, "utf8");
entry.sha256 = sidecar.split(/\s+/)[0];
const cm = sidecar.match(/(\d+) engrams/);
if (cm) entry.count = parseInt(cm[1], 10);
const tm = sidecar.match(/taken_at (\S+)/);
if (tm) entry.takenAt = tm[1];
} catch {
}
out.push(entry);
}
return out.sort((a, b) => a.stamp < b.stamp ? 1 : -1);
}
function rotate(root, now) {
const all = listBackups(root);
if (all.length <= KEEP_DAILY) return;
const keep = /* @__PURE__ */ new Set();
for (const b of all.slice(0, KEEP_DAILY)) keep.add(b.path);
const weeksSeen = /* @__PURE__ */ new Set();
for (const b of all.slice(KEEP_DAILY)) {
const week = isoWeek(/* @__PURE__ */ new Date(`${b.stamp}T00:00:00Z`));
if (weeksSeen.has(week)) continue;
weeksSeen.add(week);
if (weeksSeen.size <= KEEP_WEEKLY) keep.add(b.path);
}
for (const b of all) {
if (keep.has(b.path)) continue;
try {
fs.unlinkSync(b.path);
fs.unlinkSync(`${b.path}.sha256`);
} catch {
}
}
void now;
}
function isoWeek(d) {
const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
const day = t.getUTCDay() || 7;
t.setUTCDate(t.getUTCDate() + 4 - day);
const yearStart = new Date(Date.UTC(t.getUTCFullYear(), 0, 1));
const week = Math.ceil(((t.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
return `${t.getUTCFullYear()}-W${week}`;
}
function planRestore(root, storePath, stamp) {
const all = listBackups(root);
if (all.length === 0) throw new Error(`[plur] no backups found in ${path.join(root, BACKUP_DIR)}`);
const backup = stamp ? all.find((b) => b.stamp === stamp) : all[0];
if (!backup) throw new Error(`[plur] no backup for ${stamp}. Available: ${all.map((b) => b.stamp).join(", ")}`);
const bytes = fs.readFileSync(backup.path);
const actualSha256 = sha256(bytes);
const integrityOk = backup.sha256 === void 0 ? false : backup.sha256 === actualSha256;
const validity = validateStore(backup.path);
const backupIds = new Set(idsIn(backup.path));
const currentIds = idsIn(storePath);
const wouldLose = currentIds.filter((id) => !backupIds.has(id));
return {
backup,
validity,
actualSha256,
integrityOk,
wouldLose,
// Compare against the snapshot's INSTANT where we have it. Falling back to
// the end of its day is the conservative direction when a sidecar predates
// this field: it under-reports rather than inventing losses.
unrecoverable: idsCreatedAfter(root, backup.takenAt ?? `${backup.stamp}T23:59:59.999Z`).filter((id) => !backupIds.has(id))
};
}
function idsIn(filePath) {
try {
const doc = yaml.load(fs.readFileSync(filePath, "utf8"));
if (!doc || !Array.isArray(doc.engrams)) return [];
return doc.engrams.map((e) => e?.id).filter((id) => typeof id === "string");
} catch {
return [];
}
}
function idsCreatedAfter(root, since) {
const dir = path.join(root, "history");
if (!fs.existsSync(dir)) return [];
const ids = [];
for (const name of fs.readdirSync(dir)) {
if (!name.endsWith(".jsonl")) continue;
let lines;
try {
lines = fs.readFileSync(path.join(dir, name), "utf8").split("\n");
} catch {
continue;
}
for (const line of lines) {
if (!line.trim()) continue;
try {
const ev = JSON.parse(line);
if (typeof ev?.timestamp !== "string" || ev.timestamp <= since) continue;
if (typeof ev?.engram_id === "string") ids.push(ev.engram_id);
} catch {
}
}
}
return [...new Set(ids)];
}
function restoreBackup(root, storePath, opts = {}) {
let plan;
const superseded = `${storePath}.superseded-${Date.now()}`;
withLock(storePath, () => {
plan = planRestore(root, storePath, opts.stamp);
if (!opts.force) {
const problems = [];
if (!plan.validity.ok) problems.push(...plan.validity.reasons);
if (!plan.integrityOk) {
problems.push(
plan.backup.sha256 === void 0 ? "no sha256 sidecar \u2014 cannot verify the backup is intact" : "sha256 does not match the sidecar \u2014 the backup itself is damaged"
);
}
if (problems.length > 0) {
throw new Error(
`[plur] refusing to restore ${plan.backup.path}: ${problems.join("; ")}.
Restoring is a whole-corpus overwrite; doing it from a backup that does not verify would replace a damaged store with a differently damaged one.
Pass force to override if you have inspected the file yourself.`
);
}
}
if (fs.existsSync(storePath)) {
fs.copyFileSync(storePath, superseded);
flushFileAt(superseded);
}
atomicWrite(storePath, fs.readFileSync(plan.backup.path, "utf8"));
});
if (plan.wouldLose.length > 0) {
logger.warning(
`[plur:restore] ${plan.wouldLose.length} engram(s) present before the restore are not in this backup: ${plan.wouldLose.slice(0, 10).join(", ")}${plan.wouldLose.length > 10 ? ", \u2026" : ""}. The pre-restore store was kept at ${superseded}.`
);
}
if (plan.unrecoverable.length > 0) {
logger.warning(
`[plur:restore] history records ${plan.unrecoverable.length} engram(s) created after this backup that it does not contain: ${plan.unrecoverable.slice(0, 10).join(", ")}${plan.unrecoverable.length > 10 ? ", \u2026" : ""}.`
);
}
return { ...plan, restored: true, supersededPath: superseded };
}
// src/history.ts
import * as fs2 from "fs";
import { join as join2 } from "path";
import { createHash as createHash2 } from "crypto";
function appendHistory(root, event) {
const historyDir = join2(root, "history");
if (!fs2.existsSync(historyDir)) {
fs2.mkdirSync(historyDir, { recursive: true });
}
const date = event.timestamp.slice(0, 7);
const filePath = join2(historyDir, `${date}.jsonl`);
const line = JSON.stringify(event) + "\n";
const fd = fs2.openSync(filePath, "a");
try {
fs2.writeSync(fd, line);
try {
fs2.fsyncSync(fd);
} catch {
}
} finally {
fs2.closeSync(fd);
}
}
function readHistory(root, yearMonth) {
const filePath = join2(root, "history", `${yearMonth}.jsonl`);
if (!fs2.existsSync(filePath)) return [];
const content = fs2.readFileSync(filePath, "utf8");
const lines = content.split("\n").filter((l) => l.trim().length > 0);
const events = [];
for (const line of lines) {
try {
events.push(JSON.parse(line));
} catch {
}
}
return events;
}
function listHistoryMonths(root) {
const historyDir = join2(root, "history");
if (!fs2.existsSync(historyDir)) return [];
return fs2.readdirSync(historyDir).filter((f) => f.endsWith(".jsonl")).map((f) => f.replace(".jsonl", "")).sort();
}
function readHistoryForEngram(root, engramId) {
const months = listHistoryMonths(root);
const events = [];
for (const month of months) {
const monthEvents = readHistory(root, month);
for (const event of monthEvents) {
if (event.engram_id === engramId) {
events.push(event);
}
}
}
return events;
}
var _PROC_SALT = (process.pid % 1296).toString(36).padStart(2, "0");
var _evtSeq = 0;
var _injSeq = 0;
function generateEventId() {
return `EVT-${Date.now()}-${_PROC_SALT}${(_evtSeq++).toString(36).padStart(4, "0")}`;
}
function generateInjectionId() {
return `INJ-${Date.now()}-${_PROC_SALT}${(_injSeq++).toString(36).padStart(4, "0")}`;
}
function computeQueryHash(task) {
const normalized = task.toLowerCase().replace(/\s+/g, " ").trim();
return createHash2("sha256").update(normalized).digest("hex").slice(0, 16);
}
function findLatestInjectionFor(root, engramId, maxMonths = 2) {
const now = /* @__PURE__ */ new Date();
const allowed = /* @__PURE__ */ new Set();
for (let i = 0; i < maxMonths; i++) {
const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1));
allowed.add(d.toISOString().slice(0, 7));
}
const months = listHistoryMonths(root).filter((m) => allowed.has(m)).reverse();
for (const month of months) {
let latest = null;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const ids = event.data.ids;
if (!Array.isArray(ids) || !ids.includes(engramId)) continue;
if (!latest || event.timestamp > latest.timestamp) latest = event;
}
if (latest) return { injection_id: latest.engram_id, timestamp: latest.timestamp };
}
return null;
}
var INJECTION_SOURCES = /* @__PURE__ */ new Set([
"session_start",
"inject",
"hook",
"unknown"
]);
var ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
function readCoInjections(root, months) {
const events = [];
let skipped = 0;
const wanted = months ? new Set(months) : null;
for (const month of listHistoryMonths(root)) {
if (wanted && !wanted.has(month)) continue;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const raw = event.data;
if (!Array.isArray(raw.ids) || typeof raw.query_hash !== "string") {
skipped++;
continue;
}
if (typeof event.timestamp !== "string" || !ISO_TIMESTAMP.test(event.timestamp)) {
skipped++;
continue;
}
const ids = raw.ids.filter((id) => typeof id === "string" && id.length > 0);
if (ids.length !== raw.ids.length) skipped++;
const data = { ids, query_hash: raw.query_hash };
if (typeof raw.tokens_used === "number" && Number.isFinite(raw.tokens_used)) {
data.tokens_used = raw.tokens_used;
}
if (raw.source !== void 0) {
data.source = INJECTION_SOURCES.has(raw.source) ? raw.source : "unknown";
}
if (typeof raw.scope === "string") data.scope = raw.scope;
if (typeof raw.session_id === "string") data.session_id = raw.session_id;
events.push({ injection_id: event.engram_id, timestamp: event.timestamp, data });
}
}
events.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
return { events, skipped };
}
function countInjectionEvents(root) {
const counts = {
co_injection: 0,
injection_outcome: 0,
outcome_positive: 0,
outcome_negative: 0
};
for (const month of listHistoryMonths(root)) {
for (const event of readHistory(root, month)) {
if (event.event === "co_injection") {
counts.co_injection++;
} else if (event.event === "injection_outcome") {
counts.injection_outcome++;
if (event.data.signal === "positive") counts.outcome_positive++;
else if (event.data.signal === "negative") counts.outcome_negative++;
}
}
}
return counts;
}
// src/content-hash.ts
import { createHash as createHash3 } from "crypto";
function normalizeStatement(statement) {
return statement.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
}
function computeContentHash(statement) {
const normalized = normalizeStatement(statement);
return createHash3("sha256").update(normalized).digest("hex");
}
// src/dedup.ts
function buildDedupPrompt(newStatement, candidates) {
const candidateList = candidates.map(
(c, i) => `${i + 1}. [${c.id}] (${c.type}${c.domain ? ", domain: " + c.domain : ""})
"${c.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare a new memory statement against existing ones.
NEW STATEMENT:
"${newStatement}"
EXISTING ENGRAMS:
${candidateList}
For each existing engram, answer:
1. RELATIONSHIP: Is the new statement a DUPLICATE (same meaning), EVOLUTION (updated version of same knowledge), COMPLEMENTARY (related but different angle), or UNRELATED?
2. RICHNESS: Does the new statement contain more specific, actionable information than the existing one? (yes/no)
Then give your OVERALL DECISION (exactly one):
- NOOP: New statement is an exact duplicate of an existing engram. Return the ID.
- UPDATE: New statement is an evolution with MORE information. Return the ID to update.
- MERGE: New statement and an existing one are complementary \u2014 combining them preserves both. Return the ID to merge with.
- ADD: New statement is genuinely new knowledge.
Respond in this exact format:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID if UPDATE/MERGE/NOOP, or "none" if ADD>
REASON: <one sentence explanation>`;
}
function buildBatchDedupPrompt(statements, existingEngrams) {
const stmtList = statements.map((s, i) => `${i + 1}. "${s}"`).join("\n");
const engramList = existingEngrams.map(
(e, i) => `${i + 1}. [${e.id}] (${e.type}${e.domain ? ", domain: " + e.domain : ""})
"${e.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare NEW statements against existing engrams.
NEW STATEMENTS:
${stmtList}
EXISTING ENGRAMS:
${engramList}
For each NEW statement, decide:
- NOOP: Exact duplicate of an existing engram.
- UPDATE: Evolution with more info than existing.
- MERGE: Complementary with existing \u2014 combine.
- ADD: Genuinely new knowledge.
Respond with one block per new statement:
STATEMENT_1:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID or "none">
STATEMENT_2:
...`;
}
function parseDedupResponse(response) {
const decisionMatch = response.match(/DECISION:\s*(ADD|UPDATE|MERGE|NOOP)/i);
const targetMatch = response.match(/TARGET:\s*([^\n]+)/i);
const reasonMatch = response.match(/REASON:\s*([^\n]+)/i);
const decision = decisionMatch?.[1]?.toUpperCase() ?? "ADD";
const targetRaw = targetMatch?.[1]?.trim() ?? "none";
const target_id = targetRaw === "none" ? null : targetRaw.replace(/[^A-Za-z0-9-]/g, "");
const reason = reasonMatch?.[1]?.trim() ?? "";
return { decision, target_id, reason };
}
export {
ExtractionProvenanceSchema,
getExtractionProvenance,
EngramSchemaPassthrough,
BACKUP_DIR,
validateStore,
maybeDailyBackup,
listBackups,
planRestore,
restoreBackup,
appendHistory,
readHistory,
listHistoryMonths,
readHistoryForEngram,
generateEventId,
generateInjectionId,
computeQueryHash,
findLatestInjectionFor,
readCoInjections,
countInjectionEvents,
normalizeStatement,
computeContentHash,
buildDedupPrompt,
buildBatchDedupPrompt,
parseDedupResponse
};
import {
engramSearchText
} from "./chunk-SYCJM6JJ.js";
import {
atomicWrite
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/embeddings.ts
import { existsSync, readFileSync, mkdirSync } from "fs";
import { join, dirname } from "path";
import { createHash } from "crypto";
var EMBED_DIM = 384;
var embedPipeline = null;
var lastLoadError = null;
var transformersUnavailable = false;
function readDisabledFromEnv(env) {
const raw = env.PLUR_DISABLE_EMBEDDINGS;
if (!raw) return null;
const normalized = raw.trim().toLowerCase();
if (normalized === "1" || normalized === "true" || normalized === "yes") {
return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var";
}
return null;
}
var ENV_DISABLED_REASON = readDisabledFromEnv(process.env);
var embeddingsDisabled = ENV_DISABLED_REASON !== null;
var disabledReason = ENV_DISABLED_REASON;
function embedderStatus() {
return {
available: !embeddingsDisabled && !transformersUnavailable,
loaded: embedPipeline !== null,
lastError: lastLoadError,
disabled: embeddingsDisabled,
disabledReason
};
}
function setEmbeddingsEnabled(enabled, reason) {
embeddingsDisabled = !enabled;
disabledReason = enabled ? null : reason ?? "embeddings disabled by config";
if (!enabled) {
embedPipeline = null;
}
}
function resetEmbedder() {
transformersUnavailable = false;
lastLoadError = null;
embedPipeline = null;
}
function _setCachedEmbedder(adapter) {
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
}
async function getEmbedder() {
if (embeddingsDisabled) return null;
if (embedPipeline) return embedPipeline;
try {
const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-TB252LRE.js");
const adapter = getAdapter(resolveEmbedderName());
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
return embedPipeline;
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
return null;
}
}
async function embed(text, role) {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.embed === "function") {
let vector;
try {
vector = await embedder.embed(text, role);
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
embedPipeline = null;
return null;
}
if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) {
throw new Error(
`Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.`
);
}
return vector;
}
const result = await embedder(text, { pooling: "cls", normalize: true });
return new Float32Array(result.data);
}
async function getActiveEmbedderMeta() {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.name === "string" && typeof embedder.dim === "number") {
return { name: embedder.name, dim: embedder.dim };
}
return { name: "legacy-pipeline", dim: 0 };
}
async function activeEmbedderDim() {
const meta = await getActiveEmbedderMeta();
return meta && meta.dim > 0 ? meta.dim : null;
}
function cosineSimilarity(a, b) {
let dot = 0;
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
return dot;
}
var CACHE_VERSION = 1;
function emptyCache(meta) {
return {
meta: {
embedder_name: meta.name,
embedder_dim: meta.dim,
version: CACHE_VERSION
},
entries: {}
};
}
function loadCache(cachePath, active) {
if (!existsSync(cachePath)) return emptyCache(active);
try {
const raw = JSON.parse(readFileSync(cachePath, "utf8"));
if (!raw || typeof raw !== "object" || !raw.meta) {
logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`);
return emptyCache(active);
}
const meta = raw.meta;
if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) {
logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`);
return emptyCache(active);
}
const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {};
return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries };
} catch {
return emptyCache(active);
}
}
function saveCache(cachePath, cache) {
const dir = dirname(cachePath);
if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true });
atomicWrite(cachePath, JSON.stringify(cache), { durable: false });
}
function hashStatement(statement) {
return createHash("sha256").update(statement).digest("hex").slice(0, 16);
}
async function embeddingSearch(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const score = cosineSimilarity(queryEmbedding, engramEmbedding);
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit).map((s) => s.engram);
}
async function embeddingSearchWithScores(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding);
const score = Math.max(0, Math.min(1, rawScore));
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit);
}
async function rebuildJsonCache(engrams, storagePath, opts) {
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) {
return { reembedded: 0, skipped: true, reason: "embedder unavailable" };
}
const cachePath = join(storagePath, ".embeddings-cache.json");
const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta);
let count = 0;
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue;
const vec = await embed(searchText);
if (!vec) {
return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" };
}
cache.entries[engram.id] = { hash, embedding: Array.from(vec) };
count++;
}
saveCache(cachePath, cache);
return { reembedded: count, skipped: false };
}
export {
EMBED_DIM,
readDisabledFromEnv,
embedderStatus,
setEmbeddingsEnabled,
resetEmbedder,
_setCachedEmbedder,
embed,
activeEmbedderDim,
cosineSimilarity,
embeddingSearch,
embeddingSearchWithScores,
rebuildJsonCache
};
import {
DEFAULT_EMBEDDER,
EMBEDDER_NAMES,
_resetEmbedderCache,
_resetResolveWarnings,
getEmbedder,
resolveEmbedderName
} from "./chunk-TWFYTLOE.js";
import "./chunk-E4YVUWMJ.js";
export {
DEFAULT_EMBEDDER,
EMBEDDER_NAMES,
_resetEmbedderCache,
_resetResolveWarnings,
getEmbedder,
resolveEmbedderName
};
import {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
} from "./chunk-W56Y5QPY.js";
import "./chunk-SYCJM6JJ.js";
import "./chunk-TXHLQGN3.js";
import "./chunk-E4YVUWMJ.js";
export {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
};
import {
MIN_TOKEN_LENGTH,
TOKENIZER_VERSION,
computeIdf,
embeddingContentHash,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
hashEmbeddedText,
searchEngrams,
termMatches
} from "./chunk-SYCJM6JJ.js";
export {
MIN_TOKEN_LENGTH,
TOKENIZER_VERSION,
computeIdf,
embeddingContentHash,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
hashEmbeddedText,
searchEngrams,
termMatches
};
import {
appendHistory,
buildDedupPrompt,
computeContentHash,
maybeDailyBackup,
parseDedupResponse
} from "./chunk-VU5HJWBU.js";
import {
withAsyncLock
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/learn-async.ts
async function persistOne(deps, corpus, changed) {
if (deps.store.updateMany) {
await deps.store.updateMany([changed]);
deps.store.invalidate();
return;
}
await deps.store.save(corpus);
}
async function withStoreLock(deps, fn) {
const guarded = async () => {
try {
maybeDailyBackup(deps.rootPath, deps.engramsPath);
} catch {
}
return await fn();
};
if (deps.store.withExclusiveAccess) return await deps.store.withExclusiveAccess(guarded);
return await withAsyncLock(deps.engramsPath, guarded);
}
function demoteIfSensitive(deps, engram, newStatement) {
const tags = Array.isArray(engram.tags) ? engram.tags.filter((t) => typeof t === "string") : [];
const scanText = tags.length ? `${newStatement}
${tags.join(" ")}` : newStatement;
const offending = deps.offendingHitsForScope(scanText, engram.scope ?? "global");
if (offending.length === 0) return;
const patterns = [...new Set(offending.map((h) => h.pattern))].join(", ");
logger.warning(
`[plur] sensitive content (${patterns}) held back from shared scope "${engram.scope}" \u2014 demoted to local/private so it is not written to a shared store. Re-scope deliberately if this is a false positive.`
);
const from = engram.scope ?? "global";
engram.scope = "local";
engram.visibility = "private";
engram.structured_data = {
...engram.structured_data ?? {},
_demoted: { from, to: "local", patterns }
};
}
async function executeDedupDecision(deps, statement, context, decision, targetId) {
switch (decision) {
case "NOOP": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing) return { engram: existing, decision: "NOOP", existing_id: targetId };
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "UPDATE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const updated = { ...engrams[idx] };
updated.statement = statement;
updated.content_hash = computeContentHash(statement);
updated.engram_version = (updated.engram_version ?? 1) + 1;
updated.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) updated.tags = [.../* @__PURE__ */ new Set([...updated.tags, ...context.tags])];
demoteIfSensitive(deps, updated, updated.statement);
engrams[idx] = updated;
await persistOne(deps, engrams, updated);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_updated",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { old_statement: existing.statement, new_statement: statement, reason: "LLM dedup UPDATE" }
});
return { engram: updated, decision: "UPDATE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "MERGE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const merged = { ...engrams[idx] };
merged.statement = `${merged.statement} ${statement}`;
merged.content_hash = computeContentHash(merged.statement);
merged.engram_version = (merged.engram_version ?? 1) + 1;
merged.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) merged.tags = [.../* @__PURE__ */ new Set([...merged.tags, ...context.tags])];
if (0.7 > merged.activation.retrieval_strength) merged.activation.retrieval_strength = 0.7;
demoteIfSensitive(deps, merged, merged.statement);
engrams[idx] = merged;
await persistOne(deps, engrams, merged);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_merged",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { merged_statement: statement, reason: "LLM dedup MERGE" }
});
return { engram: merged, decision: "MERGE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "ADD":
default:
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
}
async function learnAsync(deps, statement, context) {
const hashMatch = await deps.hashDedup(statement, context?.scope);
if (hashMatch) {
return { engram: hashMatch, decision: "NOOP", existing_id: hashMatch.id };
}
const { enabled = true, threshold = 0.85, mode = "llm" } = deps.dedupConfig;
if (!enabled || mode === "off") {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
let candidates = [];
try {
candidates = await deps.recallHybrid(statement, { limit: 5 });
} catch {
candidates = await deps.recall(statement, { limit: 5 });
}
if (candidates.length === 0) {
candidates = await deps.recall(statement, { limit: 5 });
}
candidates = candidates.filter((c) => c.status === "active");
if (context?.scope) {
candidates = candidates.filter((c) => c.scope === context.scope);
}
if (candidates.length === 0) {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
const llm = context?.llm;
let decision = "ADD";
let targetId = null;
if (mode === "llm" && llm && deps.isLlmAvailable()) {
try {
const prompt = buildDedupPrompt(
statement,
candidates.map((c) => ({ id: c.id, statement: c.statement, type: c.type, domain: c.domain }))
);
const response = await llm(prompt);
const parsed = parseDedupResponse(response);
decision = parsed.decision;
targetId = parsed.target_id;
deps.recordLlmSuccess();
} catch (err) {
logger.warning(`LLM dedup failed, falling back to cosine: ${err}`);
deps.recordLlmFailure();
decision = "ADD";
}
}
return executeDedupDecision(deps, statement, context, decision, targetId);
}
async function learnBatch(deps, statements, llm, opts = {}) {
const results = [];
const failures = [];
const stats = { added: 0, updated: 0, merged: 0, noops: 0, failed: 0 };
const maxLlmCalls = opts.maxLlmCalls ?? 50;
let llmCallsUsed = 0;
let capWarned = false;
for (let i = 0; i < statements.length; i++) {
const { statement, context } = statements[i];
const stmtLlm = context?.llm ?? llm;
let effectiveLlm = stmtLlm;
if (stmtLlm) {
if (llmCallsUsed >= maxLlmCalls) {
effectiveLlm = void 0;
if (!capWarned) {
logger.warning(`learnBatch: maxLlmCalls (${maxLlmCalls}) reached \u2014 remaining statements use cosine/ADD dedup`);
capWarned = true;
}
} else {
effectiveLlm = async (prompt) => {
llmCallsUsed++;
return stmtLlm(prompt);
};
}
}
const ctx = { ...context, llm: effectiveLlm };
try {
const result = await learnAsync(deps, statement, ctx);
results.push({ ...result, input_index: i });
const key = result.decision.toLowerCase();
if (key === "noop") stats.noops++;
else if (key === "update") stats.updated++;
else if (key === "merge") stats.merged++;
else stats.added++;
} catch (err) {
stats.failed++;
failures.push({ index: i, statement, error: err instanceof Error ? err.message : String(err) });
logger.warning(`learnBatch: statement ${i} failed \u2014 ${err instanceof Error ? err.message : String(err)}`);
}
}
return { results, stats, failures };
}
export {
learnAsync,
learnBatch
};

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display