@plur-ai/core
Advanced tools
| // src/fts.ts | ||
| import { createHash } from "crypto"; | ||
| var STOP_WORDS = /* @__PURE__ */ new Set([ | ||
| "the", | ||
| "and", | ||
| "for", | ||
| "that", | ||
| "this", | ||
| "with", | ||
| "from", | ||
| "are", | ||
| "was", | ||
| "were", | ||
| "been", | ||
| "have", | ||
| "has", | ||
| "not", | ||
| "but", | ||
| "its", | ||
| "you", | ||
| "your", | ||
| "can", | ||
| "will", | ||
| "should", | ||
| "would", | ||
| "could", | ||
| "may", | ||
| "might" | ||
| ]); | ||
| var MIN_TOKEN_LENGTH = 2; | ||
| var TOKENIZER_VERSION = 4; | ||
| var SPACELESS_RUN = /[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Thai}\p{Script_Extensions=Khmer}\p{Script_Extensions=Lao}\p{Script_Extensions=Myanmar}]{2,}/gu; | ||
| var MAX_SPACELESS_RUN_CHARS = 512; | ||
| var DENSE_SCRIPT = /[\p{Script_Extensions=Hangul}\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}]/u; | ||
| function ftsTokenize(text) { | ||
| const lower = text.toLowerCase(); | ||
| const wordSource = lower.replace(SPACELESS_RUN, " ").replace(/[^\p{L}\p{N}\p{M}_\s]/gu, " "); | ||
| const tokens = wordSource.split(/\s+/).filter((w) => w.length > 2 || w.length === 2 && DENSE_SCRIPT.test(w)).filter((w) => !STOP_WORDS.has(w)); | ||
| for (const run of lower.match(SPACELESS_RUN) ?? []) { | ||
| const span = Math.min(run.length, MAX_SPACELESS_RUN_CHARS); | ||
| for (let i = 0; i < span - 1; i++) tokens.push(run.slice(i, i + 2)); | ||
| } | ||
| return tokens; | ||
| } | ||
| function searchTextFrom(fields) { | ||
| return engramSearchText({ | ||
| ...fields, | ||
| tags: fields.tags ?? [], | ||
| domain: fields.domain ?? void 0, | ||
| rationale: fields.rationale ?? void 0, | ||
| source: fields.source ?? void 0 | ||
| }); | ||
| } | ||
| function engramSearchText(engram) { | ||
| const parts = [engram.statement]; | ||
| if (engram.domain) parts.push(engram.domain.replace(/\./g, " ")); | ||
| if (engram.tags.length > 0) parts.push(engram.tags.join(" ")); | ||
| if (engram.entities) { | ||
| for (const e of engram.entities) { | ||
| parts.push(e.name); | ||
| if (e.type !== "other") parts.push(e.type); | ||
| } | ||
| } | ||
| if (engram.temporal) { | ||
| if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from); | ||
| if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until); | ||
| } | ||
| if (engram.rationale) parts.push(engram.rationale); | ||
| if (engram.source) parts.push(engram.source); | ||
| if (engram.dual_coding) { | ||
| if (engram.dual_coding.example) parts.push(engram.dual_coding.example); | ||
| if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy); | ||
| } | ||
| if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) { | ||
| for (const a of engram.knowledge_anchors) { | ||
| if (a.snippet) parts.push(a.snippet); | ||
| } | ||
| } | ||
| return parts.join(" "); | ||
| } | ||
| function embeddingContentHash(engram) { | ||
| return hashEmbeddedText(engramSearchText(engram)); | ||
| } | ||
| function hashEmbeddedText(text) { | ||
| return createHash("md5").update(text).digest("hex"); | ||
| } | ||
| function termMatches(t, qt) { | ||
| return t.includes(qt) || qt.startsWith(t); | ||
| } | ||
| function computeIdf(engrams, queryTokens, stats) { | ||
| if (stats) { | ||
| if (stats.N === 0) return /* @__PURE__ */ new Map(); | ||
| const idf2 = /* @__PURE__ */ new Map(); | ||
| for (const qt of queryTokens) { | ||
| const df = stats.df.get(qt) ?? 0; | ||
| idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df)))); | ||
| } | ||
| return idf2; | ||
| } | ||
| const N = engrams.length; | ||
| if (N === 0) return /* @__PURE__ */ new Map(); | ||
| const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e)))); | ||
| const idf = /* @__PURE__ */ new Map(); | ||
| for (const qt of queryTokens) { | ||
| let df = 0; | ||
| for (const termSet of engramTermSets) { | ||
| if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) { | ||
| df++; | ||
| } | ||
| } | ||
| idf.set(qt, Math.max(0, Math.log(N / (1 + df)))); | ||
| } | ||
| return idf; | ||
| } | ||
| function extendCorpusStats(stats, queryTokens, outsiders) { | ||
| if (outsiders.length === 0) return stats; | ||
| const termSets = []; | ||
| let totalLen = 0; | ||
| for (const e of outsiders) { | ||
| const terms = ftsTokenize(engramSearchText(e)); | ||
| totalLen += terms.length; | ||
| termSets.push(new Set(terms)); | ||
| } | ||
| const df = new Map(stats.df); | ||
| for (const qt of queryTokens) { | ||
| let added = 0; | ||
| for (const set of termSets) { | ||
| if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++; | ||
| } | ||
| if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added); | ||
| } | ||
| const N = stats.N + outsiders.length; | ||
| return { | ||
| N, | ||
| df, | ||
| avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0 | ||
| }; | ||
| } | ||
| var BM25_K1 = 1.2; | ||
| var BM25_B = 0.75; | ||
| function ftsScore(engram, queryTokens, idfWeights, avgDocLength) { | ||
| const allTerms = ftsTokenize(engramSearchText(engram)); | ||
| if (queryTokens.length === 0) return 0; | ||
| const docLen = allTerms.length; | ||
| const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen; | ||
| const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0); | ||
| let score = 0; | ||
| for (const qt of queryTokens) { | ||
| let effectiveIdf; | ||
| if (!idfWeights) { | ||
| effectiveIdf = 1; | ||
| } else if (hasNonZeroIdf) { | ||
| effectiveIdf = idfWeights.get(qt) ?? 0; | ||
| if (effectiveIdf === 0) continue; | ||
| } else { | ||
| effectiveIdf = 1; | ||
| } | ||
| let tf = 0; | ||
| for (const t of allTerms) { | ||
| if (termMatches(t, qt)) tf++; | ||
| } | ||
| if (tf === 0) continue; | ||
| const numerator = tf * (BM25_K1 + 1); | ||
| const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl); | ||
| score += effectiveIdf * (numerator / denominator); | ||
| } | ||
| return score; | ||
| } | ||
| function searchEngrams(engrams, query, limit = 20, stats) { | ||
| const queryTokens = ftsTokenize(query); | ||
| if (queryTokens.length === 0) return []; | ||
| const idfWeights = computeIdf(engrams, queryTokens, stats); | ||
| const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0; | ||
| let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0); | ||
| if (scored.length === 0) { | ||
| scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0); | ||
| } | ||
| return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram); | ||
| } | ||
| export { | ||
| MIN_TOKEN_LENGTH, | ||
| TOKENIZER_VERSION, | ||
| MAX_SPACELESS_RUN_CHARS, | ||
| ftsTokenize, | ||
| searchTextFrom, | ||
| engramSearchText, | ||
| embeddingContentHash, | ||
| hashEmbeddedText, | ||
| termMatches, | ||
| computeIdf, | ||
| extendCorpusStats, | ||
| ftsScore, | ||
| searchEngrams | ||
| }; |
| import { | ||
| atomicWrite, | ||
| withLock | ||
| } from "./chunk-TXHLQGN3.js"; | ||
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/schemas/engram.ts | ||
| import { z } from "zod"; | ||
| var ActivationSchema = z.object({ | ||
| retrieval_strength: z.number().min(0).max(1), | ||
| storage_strength: z.number().min(0).max(1), | ||
| frequency: z.number().int().min(0), | ||
| last_accessed: z.string().describe("Date or ISO 8601 timestamp of last access.") | ||
| }).describe("ACT-R activation parameters driving decay and ranking. STABLE."); | ||
| var KnowledgeTypeSchema = z.object({ | ||
| memory_class: z.enum(["semantic", "episodic", "procedural", "metacognitive"]), | ||
| cognitive_level: z.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"]).describe("Bloom's taxonomy level.") | ||
| }); | ||
| var KnowledgeAnchorSchema = z.object({ | ||
| path: z.string().describe("Path to a grounding document/file."), | ||
| relevance: z.enum(["primary", "supporting", "example"]).default("supporting"), | ||
| snippet: z.string().max(200).optional(), | ||
| snippet_extracted_at: z.string().optional() | ||
| }); | ||
| var AssociationSchema = z.object({ | ||
| target_type: z.enum(["engram", "document"]), | ||
| target: z.string().describe("ID or path of the association target."), | ||
| strength: z.number().min(0).max(0.95), | ||
| type: z.enum(["semantic", "temporal", "causal", "co_accessed"]), | ||
| updated_at: z.string().optional() | ||
| }); | ||
| var DualCodingSchema = z.object({ | ||
| example: z.string().optional(), | ||
| analogy: z.string().optional() | ||
| }).describe("Worked example and/or analogy (dual coding). At least one of example or analogy MUST be provided (enforced at runtime by the Zod .refine below).").refine( | ||
| (d) => d.example || d.analogy, | ||
| "At least one of example or analogy must be provided" | ||
| ); | ||
| var RelationsSchema = z.object({ | ||
| broader: z.array(z.string()).default([]), | ||
| narrower: z.array(z.string()).default([]), | ||
| related: z.array(z.string()).default([]), | ||
| conflicts: z.array(z.string()).default([]), | ||
| /** IDs of engrams this one intentionally replaces (#240). An intentional | ||
| * update is not a tension — the scanner skips supersedes-linked pairs. */ | ||
| supersedes: z.array(z.string()).default([]), | ||
| /** Reverse edge of `supersedes` (#240) — IDs of engrams that replace this one. */ | ||
| superseded_by: z.array(z.string()).default([]) | ||
| }).describe("Typed graph edges between engram IDs."); | ||
| var ProvenanceSchema = z.object({ | ||
| origin: z.string(), | ||
| chain: z.array(z.string()).default([]), | ||
| signature: z.string().nullable().default(null).describe("RESERVED. Detached signature over the engram. Algorithm and canonicalization not yet specified \u2014 see ENGRAM-STANDARD-v1.md \xA77."), | ||
| license: z.string().default("cc-by-sa-4.0") | ||
| }).describe("Origin and signing chain. STABLE for origin/chain/license; signature is RESERVED (see ENGRAM-STANDARD-v1.md \xA77)."); | ||
| var FeedbackSignalsSchema = z.object({ | ||
| positive: z.number().int().default(0), | ||
| negative: z.number().int().default(0), | ||
| neutral: z.number().int().default(0) | ||
| }); | ||
| var EntityRefSchema = z.object({ | ||
| name: z.string(), | ||
| type: z.enum([ | ||
| "person", | ||
| "organization", | ||
| "technology", | ||
| "concept", | ||
| "project", | ||
| "tool", | ||
| "place", | ||
| "event", | ||
| "standard", | ||
| "other" | ||
| ]), | ||
| uri: z.string().url().optional() | ||
| }); | ||
| var TemporalSchema = z.object({ | ||
| learned_at: z.string(), | ||
| valid_from: z.string().optional(), | ||
| valid_until: z.string().optional(), | ||
| ingested_at: z.string().optional() | ||
| }).describe("Bi-temporal anchoring (Zep-inspired). When is this knowledge true?"); | ||
| var UsageStatsSchema = z.object({ | ||
| injections: z.number().int().default(0), | ||
| hits: z.number().int().default(0), | ||
| misses: z.number().int().default(0), | ||
| last_hit_at: z.string().optional() | ||
| }); | ||
| var EpisodicFieldsSchema = z.object({ | ||
| emotional_weight: z.number().int().min(1).max(10).default(5), | ||
| confidence: z.number().int().min(1).max(10).default(5), | ||
| trigger_context: z.string().optional(), | ||
| journal_ref: z.string().optional() | ||
| }); | ||
| var PreviousVersionRefSchema = z.object({ | ||
| event_id: z.string(), | ||
| changed_at: z.string() | ||
| }); | ||
| var ExchangeMetadataSchema = z.object({ | ||
| fitness_score: z.number().min(0).max(1).optional(), | ||
| environmental_diversity: z.number().int().default(0), | ||
| adoption_count: z.number().int().default(0), | ||
| contradiction_rate: z.number().min(0).max(1).default(0) | ||
| }); | ||
| var SerendipitySchema = z.object({ | ||
| unexpectedness: z.number().min(0).max(1), | ||
| relevance: z.number().min(0).max(1), | ||
| score: z.number().min(0).max(1) | ||
| }); | ||
| var InsightFateSchema = z.enum([ | ||
| "surfaced", | ||
| // shown in a briefing; no downstream action yet | ||
| "promoted", | ||
| // became a durable engram / zettel | ||
| "cited", | ||
| // referenced in later journal/work | ||
| "tasked", | ||
| // converted to a GTD task | ||
| "dismissed", | ||
| // user/LLM rejected it | ||
| "expired" | ||
| // decayed out of the buffer unused | ||
| ]); | ||
| var InsightFieldSchema = z.object({ | ||
| /** Which memory-stream operation produced this insight. Nightly arc: | ||
| * `distill` (episode→insight synthesis) → `consolidate` (convergent gist | ||
| * abstraction over the buffer) → `dream` (divergent REM-style recombination — | ||
| * speculative, never auto-promoted). `connect`/`emerge`/`drift` are on-demand lenses. */ | ||
| operation: z.enum(["distill", "consolidate", "dream", "connect", "emerge", "drift"]), | ||
| synthesized_at: z.string(), | ||
| /** Anti-hallucination grounding. Cited source notes live in the parent engram's | ||
| * `knowledge_anchors[]`; this flags whether the claim was verified against those | ||
| * snippets. `ungrounded` = couldn't cite sources → quarantined (`candidate`, never | ||
| * surfaced). `speculative` = a `dream`: its recombined INPUTS are cited but its | ||
| * CONCLUSION is an explicit hypothesis — surfaced only as inspiration, and (per the | ||
| * promote-requires-grounding refine below) it must be re-grounded to `verified` | ||
| * before it can be promoted to a durable engram. */ | ||
| grounding: z.enum(["verified", "unverified", "ungrounded", "speculative"]).default("unverified"), | ||
| /** The episode-log slice this insight was distilled from (evidence trail). */ | ||
| source_episode_ids: z.array(z.string()).default([]), | ||
| /** Distinct objective for connect/emerge/dream insights. */ | ||
| serendipity: SerendipitySchema.optional(), | ||
| fate: InsightFateSchema.default("surfaced"), | ||
| /** Engram id / zettel path / task id the insight became, if acted upon. */ | ||
| fate_ref: z.string().optional(), | ||
| fate_at: z.string().optional(), | ||
| /** How many briefings have surfaced this insight (acted-upon-rate denominator). */ | ||
| surfaced_count: z.number().int().min(0).default(0) | ||
| }).refine( | ||
| // Promote-requires-grounding (user rule 2026-06-15): a dream is inspiration, not | ||
| // fact. A speculative/ungrounded insight can only become a durable promotion once | ||
| // it has been re-grounded in reality (grounding=verified). | ||
| (i) => i.fate !== "promoted" || i.grounding === "verified", | ||
| { message: "A promoted insight must be grounded (grounding=verified); speculative dreams cannot be promoted until re-grounded.", path: ["grounding"] } | ||
| ); | ||
| var ExtractionProvenanceSchema = z.object({ | ||
| confidence: z.number().min(0).max(1).optional().describe("0-1 classifier confidence at extraction time. Frozen at write; distinct from feedback-derived computeConfidence() and from episodic.confidence."), | ||
| source_commit: z.string().optional().describe("Git SHA of the source repository at extraction time (reproducibility)."), | ||
| extractor_version: z.string().optional().describe("Version of the extracting CLI/tool (schema-migration handle). Complementary to the pack-level capsule producer field (#61).") | ||
| }).passthrough().describe("ETL extraction provenance convention carried in structured_data.extraction (#463). Not wired into EngramSchema."); | ||
| function getExtractionProvenance(engram) { | ||
| const extraction = engram.structured_data?.["extraction"]; | ||
| if (extraction === void 0 || extraction === null) return null; | ||
| const parsed = ExtractionProvenanceSchema.safeParse(extraction); | ||
| return parsed.success ? parsed.data : null; | ||
| } | ||
| var MeasuredUnderSchema = z.object({ | ||
| /** Model or system variant under which the measurement was taken (e.g. 'claude-opus-4', 'gpt-4o'). */ | ||
| model: z.string().optional(), | ||
| /** Source environment type (e.g. 'local-git', 'gitlab', 'bench', 'production'). */ | ||
| source_type: z.string().optional(), | ||
| /** Hardware or runtime tier (e.g. 'M3-Pro-36GB', 'A100', 'CI-runner'). */ | ||
| hardware: z.string().optional(), | ||
| /** Dataset or workload identifier (e.g. 'LongMemEval-S', 'plur-bench-2026-Q2'). */ | ||
| dataset: z.string().optional(), | ||
| /** ISO date (YYYY-MM-DD) the measurement was taken. */ | ||
| date: z.string().optional() | ||
| }).passthrough().describe("Context under which a numeric or benchmark-derived measurement was taken (#869). All fields optional \u2014 absent means unknown."); | ||
| var EngramSchema = z.object({ | ||
| // Identity | ||
| id: z.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/).describe("Unique identifier. Class prefix ENG (concrete engram), ABS (abstraction), or META (meta-engram). Canonical concrete form: ENG-YYYY-MMDD-NNN; store-namespaced form: ENG-{PREFIX}-YYYY-MMDD-NNN."), | ||
| version: z.number().int().min(1).default(2).describe("Schema-shape generation of this engram object (currently 2). Distinct from engram_version, which tracks content evolution."), | ||
| // 'active' and 'retired' are the two states any current code path assigns | ||
| // (retire via forget/dedup/supersede). 'dormant' and 'candidate' are NOT | ||
| // assigned by any code today: 'dormant' was only ever set by the batchDecay | ||
| // pass removed in #563 (decay is now a read-time property, not a materialized | ||
| // status), and 'candidate' is reserved. They are kept in the enum so stores | ||
| // written before #563 that persisted status:'dormant' still load, and so the | ||
| // status filter accepts them; do not remove without a data migration. | ||
| status: z.enum(["active", "dormant", "retired", "candidate"]).describe("Lifecycle state. Assigned values today are active/retired; dormant/candidate are legacy/reserved (see note above)."), | ||
| consolidated: z.boolean().default(false).describe("Whether this engram has been through consolidation (sleep-like batch reprocessing)."), | ||
| type: z.enum(["behavioral", "terminological", "procedural", "architectural"]).describe("Top-level classification of the knowledge."), | ||
| scope: z.string().describe("Hierarchical namespace, e.g. 'global', 'project:my-app', 'group:plur/test'. Free-form string; ':' separates scope kind from path."), | ||
| visibility: z.enum(["private", "public", "template"]).default("private").describe("Sharing posture. 'private' engrams MUST NOT be exported in packs."), | ||
| // Content | ||
| statement: z.string().min(1).describe("The assertion itself \u2014 the load-bearing content of the engram."), | ||
| rationale: z.string().optional().describe("Why this is true / why it matters."), | ||
| contraindications: z.array(z.string()).optional().describe("Conditions under which the statement does NOT apply."), | ||
| // Lineage | ||
| source: z.string().optional().describe("Free-text origin (session, document, conversation)."), | ||
| source_patterns: z.array(z.string()).optional().describe("Pattern IDs that contributed to this engram."), | ||
| derivation_count: z.number().int().min(0).default(1).describe("How many derivation steps produced this engram."), | ||
| pack: z.string().nullable().default(null).describe("Name of the pack this engram belongs to, or null."), | ||
| abstract: z.string().nullable().default(null).describe("ID of an ABS- abstraction this engram instantiates, or null."), | ||
| derived_from: z.string().nullable().default(null).describe("ID of the engram this was derived from, or null."), | ||
| // Classification | ||
| knowledge_type: KnowledgeTypeSchema.optional(), | ||
| domain: z.string().optional().describe("Dotted domain path, e.g. 'dev/testing' or 'plur.session'."), | ||
| tags: z.array(z.string()).default([]).describe("Free-form tags used for matching and retrieval."), | ||
| // Activation (ACT-R model) | ||
| activation: ActivationSchema.default({ | ||
| retrieval_strength: 0.7, | ||
| storage_strength: 1, | ||
| frequency: 0, | ||
| last_accessed: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10) | ||
| }), | ||
| // Relations & grounding | ||
| relations: RelationsSchema.optional(), | ||
| associations: z.array(AssociationSchema).default([]), | ||
| knowledge_anchors: z.array(KnowledgeAnchorSchema).default([]), | ||
| dual_coding: DualCodingSchema.optional(), | ||
| // Provenance | ||
| provenance: ProvenanceSchema.optional(), | ||
| // Feedback | ||
| feedback_signals: FeedbackSignalsSchema.default({ positive: 0, negative: 0, neutral: 0 }), | ||
| // === NEW OPTIONAL FIELDS (v2.1) === | ||
| /** Typed entity references extracted from statement. Enables graph queries. */ | ||
| entities: z.array(EntityRefSchema).optional().describe("Typed entity references extracted from statement. Enables graph queries."), | ||
| /** Temporal validity window. When is this knowledge true? */ | ||
| temporal: TemporalSchema.optional(), | ||
| /** Automatic usage tracking. Injections, hits, misses. */ | ||
| usage: UsageStatsSchema.optional(), | ||
| /** Episodic context: emotional weight, confidence, trigger. */ | ||
| episodic: EpisodicFieldsSchema.optional(), | ||
| /** Exchange marketplace metadata: fitness, adoption, diversity. */ | ||
| exchange: ExchangeMetadataSchema.optional(), | ||
| /** Extensible key-value data for domain-specific fields. */ | ||
| structured_data: z.record(z.string(), z.unknown()).optional().describe("Extensible key-value data for domain-specific fields."), | ||
| /** Memory-stream insight provenance (metacognition Phase 1). Orthogonal to | ||
| * `type`. Present iff this engram was synthesized by the metacognition memory | ||
| * stream; the episodic insight buffer is the set of engrams where this is set. */ | ||
| insight: InsightFieldSchema.optional(), | ||
| /** Polarity classification: 'do' for directives, 'dont' for prohibitions, null for unclassified. */ | ||
| polarity: z.enum(["do", "dont"]).nullable().default(null).describe("'do' for directives, 'dont' for prohibitions, null for unclassified."), | ||
| // === SP1: Memory Intelligence fields === | ||
| content_hash: z.string().optional().describe("Hash of normalized statement content, used for dedup."), | ||
| commitment: z.enum(["exploring", "leaning", "decided", "locked", "draft"]).optional().describe("Commitment level of the asserted knowledge. `draft` marks an engram as pending human approval; core stores and recalls it like any other value \u2014 enforcement is left to deployments that implement a review queue. A positive feedback signal does not advance it (see feedback.ts:nextCommitment)."), | ||
| locked_at: z.string().optional().describe("Timestamp when commitment reached 'locked'."), | ||
| locked_reason: z.string().optional().describe("Why this engram was locked."), | ||
| // === SP1: Reference counting (issue #107, renamed #866) === | ||
| /** Number of learn() write attempts that resolved to this engram. | ||
| * Incremented on every hash-dedup hit; decremented by forget(). | ||
| * Engram physically retires only when this reaches 0. | ||
| * Renamed from reference_count (#866) — backfill runs on first parse of old stores. */ | ||
| write_count: z.number().int().min(0).default(1).describe("Number of learn() write attempts that resolved to this engram (same-scope re-learns). Engram retires only when this reaches 0. Renamed from reference_count (#866)."), | ||
| /** Number of times this engram was selected into a session's injection context. | ||
| * Distinct from activation.frequency (which counts recall() retrieval events, not | ||
| * inject() selections). High injection_count + low feedback_signals.positive is a | ||
| * signal for #865 efficacy detection. Incremented in the inject path (#866). */ | ||
| injection_count: z.number().int().min(0).default(0).describe("Number of times this engram was injected into session context. Distinct from activation.frequency (recall events). Use with feedback_signals for efficacy auditing (#866)."), | ||
| /** Provenance of each write attempt. One entry per write (including the | ||
| * first). Migrated old engrams without this field start with []. */ | ||
| sources: z.array(z.object({ | ||
| scope: z.string(), | ||
| session_id: z.string().nullable().default(null), | ||
| stored_at: z.string().describe("ISO 8601 timestamp of this write.") | ||
| })).default([]).describe("Provenance of each write attempt; one entry per write."), | ||
| // === SP1: Cross-scope recurrence (issue #176) === | ||
| /** Number of times this engram's content was re-learned at a DIFFERENT | ||
| * scope than the original. Triggers auto-broadening + commitment | ||
| * escalation when threshold is crossed. Distinct from write_count | ||
| * (which counts re-learns in the SAME scope) — recurrence_count is | ||
| * evidence of universal applicability, not just repetition. */ | ||
| recurrence_count: z.number().int().min(0).default(0).describe("Number of times this content was re-learned at a DIFFERENT scope than the original. Evidence of universal applicability."), | ||
| // === SP2: History & Evolution fields === | ||
| engram_version: z.number().int().min(1).default(1).describe("Content-evolution version (incremented when the statement materially changes)."), | ||
| previous_version_ref: PreviousVersionRefSchema.optional(), | ||
| episode_ids: z.array(z.string()).default([]).describe("IDs of episodes (raw conversational events) that produced or reinforced this engram."), | ||
| // === SP3: Retrieval & Injection fields === | ||
| summary: z.string().max(80).optional().describe("Short (<=80 char) injection-friendly summary."), | ||
| /** | ||
| * Always-load flag. Pinned engrams bypass the term-hits gate in scoreEngram | ||
| * and are eligible for injection on every session start, regardless of | ||
| * keyword overlap with the user's task. Use sparingly: meta-rules, | ||
| * cross-cutting safety conventions, and core operating principles only. | ||
| * Pinned engrams still respect the token budget — they bypass per-pack and | ||
| * per-domain fairness caps in fillTokenBudget so always-load behavior is | ||
| * honored even if a single pack contributes many. | ||
| */ | ||
| pinned: z.boolean().optional().describe("Always-load flag. Pinned engrams bypass the keyword-relevance gate and are eligible for injection every session. Use sparingly."), | ||
| /** Measurement context for numeric or benchmark-derived claims (#869). | ||
| * Records model, source_type, hardware, dataset, and/or date under which the | ||
| * asserted value was measured, so differing-condition measurements can be | ||
| * stored as refinements rather than tensions (#203). Absent for non-numeric | ||
| * engrams; all sub-fields are optional even when the object is present. */ | ||
| measured_under: MeasuredUnderSchema.optional() | ||
| }); | ||
| var EngramSchemaPassthrough = EngramSchema.passthrough(); | ||
| // src/backup.ts | ||
| import * as fs from "fs"; | ||
| import * as path from "path"; | ||
| import { createHash } from "crypto"; | ||
| import * as yaml from "js-yaml"; | ||
| var BACKUP_DIR = "backups"; | ||
| var KEEP_DAILY = 7; | ||
| var KEEP_WEEKLY = 4; | ||
| var SHRINK_TOLERANCE = 0.1; | ||
| function statePath(root) { | ||
| return path.join(root, BACKUP_DIR, ".state.json"); | ||
| } | ||
| function readState(root) { | ||
| try { | ||
| return JSON.parse(fs.readFileSync(statePath(root), "utf8")); | ||
| } catch { | ||
| return {}; | ||
| } | ||
| } | ||
| function writeState(root, state) { | ||
| const p = statePath(root); | ||
| fs.mkdirSync(path.dirname(p), { recursive: true }); | ||
| fs.writeFileSync(p, JSON.stringify(state, null, 2) + "\n", "utf8"); | ||
| } | ||
| function sha256(content) { | ||
| return createHash("sha256").update(content).digest("hex"); | ||
| } | ||
| function validateStore(filePath, lastGoodCount) { | ||
| const failures = []; | ||
| const reasons = []; | ||
| let raw; | ||
| try { | ||
| raw = fs.readFileSync(filePath); | ||
| } catch (err) { | ||
| return { ok: false, failures: ["unreadable"], reasons: [`cannot read ${filePath}: ${err}`], count: null }; | ||
| } | ||
| if (raw.length === 0) { | ||
| return { ok: false, failures: ["empty"], reasons: ["file is 0 bytes"], count: null }; | ||
| } | ||
| if (!raw.toString("utf8").endsWith("\n")) { | ||
| failures.push("truncated"); | ||
| reasons.push("file does not end with a newline \u2014 PLUR's writer always emits one, so this looks cut short"); | ||
| } | ||
| let doc; | ||
| try { | ||
| doc = yaml.load(raw.toString("utf8")); | ||
| } catch (err) { | ||
| return { ok: false, failures: ["unparseable"], reasons: [`YAML parse failed: ${err}`], count: null }; | ||
| } | ||
| if (doc == null || typeof doc !== "object" || Array.isArray(doc) || !Array.isArray(doc.engrams)) { | ||
| return { | ||
| ok: false, | ||
| failures: ["not-a-store"], | ||
| reasons: ["parsed, but is not a mapping with an `engrams` list"], | ||
| count: null | ||
| }; | ||
| } | ||
| const entries = doc.engrams; | ||
| const count = entries.length; | ||
| let invalid = 0; | ||
| const ids = /* @__PURE__ */ new Set(); | ||
| let duplicateIds = 0; | ||
| let missingIds = 0; | ||
| for (const entry of entries) { | ||
| if (!EngramSchemaPassthrough.safeParse(entry).success) invalid++; | ||
| const id = entry?.id; | ||
| if (typeof id !== "string" || id.length === 0) missingIds++; | ||
| else if (ids.has(id)) duplicateIds++; | ||
| else ids.add(id); | ||
| } | ||
| if (invalid > 0) { | ||
| failures.push("invalid-entries"); | ||
| reasons.push(`${invalid} entry/entries fail schema validation`); | ||
| } | ||
| if (missingIds > 0) { | ||
| failures.push("missing-ids"); | ||
| reasons.push(`${missingIds} entry/entries have no id`); | ||
| } | ||
| if (duplicateIds > 0) { | ||
| failures.push("duplicate-ids"); | ||
| reasons.push(`${duplicateIds} duplicate id(s)`); | ||
| } | ||
| if (typeof lastGoodCount === "number" && lastGoodCount > 0) { | ||
| const floor = lastGoodCount * (1 - SHRINK_TOLERANCE); | ||
| if (count < floor) { | ||
| failures.push("shrunk"); | ||
| reasons.push( | ||
| `holds ${count} engram(s) but the last good snapshot held ${lastGoodCount} \u2014 a drop this large is how a truncation looks` | ||
| ); | ||
| } | ||
| } | ||
| return { ok: failures.length === 0, failures, reasons, count }; | ||
| } | ||
| function todayStamp(now) { | ||
| return now.toISOString().slice(0, 10); | ||
| } | ||
| function snapshotPath(root, stamp) { | ||
| return path.join(root, BACKUP_DIR, `engrams-${stamp}.yaml`); | ||
| } | ||
| var doneThisProcess = /* @__PURE__ */ new Set(); | ||
| function maybeDailyBackup(root, storePath, now = /* @__PURE__ */ new Date()) { | ||
| const key = `${root}\0${todayStamp(now)}`; | ||
| if (doneThisProcess.has(key)) return { taken: false, skipped: "already-today" }; | ||
| try { | ||
| if (!fs.existsSync(storePath)) { | ||
| doneThisProcess.add(key); | ||
| return { taken: false, skipped: "no-store" }; | ||
| } | ||
| const state = readState(root); | ||
| const stamp = todayStamp(now); | ||
| if (state.last_backup_date === stamp) { | ||
| doneThisProcess.add(key); | ||
| return { taken: false, skipped: "already-today" }; | ||
| } | ||
| const existing = listBackups(root); | ||
| const strongest = existing.reduce( | ||
| (max, b) => typeof b.count === "number" && (max === void 0 || b.count > max) ? b.count : max, | ||
| void 0 | ||
| ); | ||
| const baseline = state.last_good_count ?? strongest; | ||
| const validity = validateStore(storePath, baseline); | ||
| if (!validity.ok) { | ||
| logger.warning( | ||
| `[plur:backup] refusing to snapshot ${storePath} \u2014 ${validity.reasons.join("; ")}. Your last good backup is unchanged. Run 'plur doctor' to inspect.` | ||
| ); | ||
| return { taken: false, skipped: "invalid", validity }; | ||
| } | ||
| const bytes = fs.readFileSync(storePath); | ||
| const dest = snapshotPath(root, stamp); | ||
| const sameDay = existing.find((b) => b.stamp === stamp); | ||
| if (sameDay && typeof sameDay.count === "number" && (validity.count ?? 0) < sameDay.count) { | ||
| logger.warning( | ||
| `[plur:backup] keeping today's existing snapshot (${sameDay.count} engrams) \u2014 the live store holds ${validity.count}, and replacing a stronger snapshot with a weaker one would discard the better copy. Run 'plur restore --list' to inspect.` | ||
| ); | ||
| doneThisProcess.add(key); | ||
| return { taken: false, skipped: "invalid", validity }; | ||
| } | ||
| fs.mkdirSync(path.dirname(dest), { recursive: true }); | ||
| writeFileDurable(dest, bytes); | ||
| writeFileDurable( | ||
| `${dest}.sha256`, | ||
| Buffer.from( | ||
| `${sha256(bytes)} ${path.basename(dest)} | ||
| ${validity.count} engrams | ||
| taken_at ${now.toISOString()} | ||
| `, | ||
| "utf8" | ||
| ) | ||
| ); | ||
| doneThisProcess.add(key); | ||
| writeState(root, { | ||
| last_backup_date: stamp, | ||
| last_good_count: validity.count ?? void 0, | ||
| last_good_sha256: sha256(bytes) | ||
| }); | ||
| rotate(root, now); | ||
| return { taken: true, path: dest, validity }; | ||
| } catch (err) { | ||
| logger.warning(`[plur:backup] snapshot failed (the write itself was unaffected): ${err}`); | ||
| return { taken: false, skipped: "invalid" }; | ||
| } | ||
| } | ||
| function flushFileAt(filePath) { | ||
| let fd; | ||
| try { | ||
| fd = fs.openSync(filePath, "r+"); | ||
| fs.fsyncSync(fd); | ||
| } catch { | ||
| } finally { | ||
| if (fd !== void 0) { | ||
| try { | ||
| fs.closeSync(fd); | ||
| } catch { | ||
| } | ||
| } | ||
| } | ||
| } | ||
| function writeFileDurable(dest, bytes) { | ||
| const fd = fs.openSync(dest, "w"); | ||
| try { | ||
| fs.writeFileSync(fd, bytes); | ||
| fs.fsyncSync(fd); | ||
| } finally { | ||
| fs.closeSync(fd); | ||
| } | ||
| } | ||
| function listBackups(root) { | ||
| const dir = path.join(root, BACKUP_DIR); | ||
| if (!fs.existsSync(dir)) return []; | ||
| const out = []; | ||
| for (const name of fs.readdirSync(dir)) { | ||
| const m = name.match(/^engrams-(\d{4}-\d{2}-\d{2})\.yaml$/); | ||
| if (!m) continue; | ||
| const full = path.join(dir, name); | ||
| const entry = { path: full, stamp: m[1], size: fs.statSync(full).size }; | ||
| try { | ||
| const sidecar = fs.readFileSync(`${full}.sha256`, "utf8"); | ||
| entry.sha256 = sidecar.split(/\s+/)[0]; | ||
| const cm = sidecar.match(/(\d+) engrams/); | ||
| if (cm) entry.count = parseInt(cm[1], 10); | ||
| const tm = sidecar.match(/taken_at (\S+)/); | ||
| if (tm) entry.takenAt = tm[1]; | ||
| } catch { | ||
| } | ||
| out.push(entry); | ||
| } | ||
| return out.sort((a, b) => a.stamp < b.stamp ? 1 : -1); | ||
| } | ||
| function rotate(root, now) { | ||
| const all = listBackups(root); | ||
| if (all.length <= KEEP_DAILY) return; | ||
| const keep = /* @__PURE__ */ new Set(); | ||
| for (const b of all.slice(0, KEEP_DAILY)) keep.add(b.path); | ||
| const weeksSeen = /* @__PURE__ */ new Set(); | ||
| for (const b of all.slice(KEEP_DAILY)) { | ||
| const week = isoWeek(/* @__PURE__ */ new Date(`${b.stamp}T00:00:00Z`)); | ||
| if (weeksSeen.has(week)) continue; | ||
| weeksSeen.add(week); | ||
| if (weeksSeen.size <= KEEP_WEEKLY) keep.add(b.path); | ||
| } | ||
| for (const b of all) { | ||
| if (keep.has(b.path)) continue; | ||
| try { | ||
| fs.unlinkSync(b.path); | ||
| fs.unlinkSync(`${b.path}.sha256`); | ||
| } catch { | ||
| } | ||
| } | ||
| void now; | ||
| } | ||
| function isoWeek(d) { | ||
| const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); | ||
| const day = t.getUTCDay() || 7; | ||
| t.setUTCDate(t.getUTCDate() + 4 - day); | ||
| const yearStart = new Date(Date.UTC(t.getUTCFullYear(), 0, 1)); | ||
| const week = Math.ceil(((t.getTime() - yearStart.getTime()) / 864e5 + 1) / 7); | ||
| return `${t.getUTCFullYear()}-W${week}`; | ||
| } | ||
| function planRestore(root, storePath, stamp) { | ||
| const all = listBackups(root); | ||
| if (all.length === 0) throw new Error(`[plur] no backups found in ${path.join(root, BACKUP_DIR)}`); | ||
| const backup = stamp ? all.find((b) => b.stamp === stamp) : all[0]; | ||
| if (!backup) throw new Error(`[plur] no backup for ${stamp}. Available: ${all.map((b) => b.stamp).join(", ")}`); | ||
| const bytes = fs.readFileSync(backup.path); | ||
| const actualSha256 = sha256(bytes); | ||
| const integrityOk = backup.sha256 === void 0 ? false : backup.sha256 === actualSha256; | ||
| const validity = validateStore(backup.path); | ||
| const backupIds = new Set(idsIn(backup.path)); | ||
| const currentIds = idsIn(storePath); | ||
| const wouldLose = currentIds.filter((id) => !backupIds.has(id)); | ||
| return { | ||
| backup, | ||
| validity, | ||
| actualSha256, | ||
| integrityOk, | ||
| wouldLose, | ||
| // Compare against the snapshot's INSTANT where we have it. Falling back to | ||
| // the end of its day is the conservative direction when a sidecar predates | ||
| // this field: it under-reports rather than inventing losses. | ||
| unrecoverable: idsCreatedAfter(root, backup.takenAt ?? `${backup.stamp}T23:59:59.999Z`).filter((id) => !backupIds.has(id)) | ||
| }; | ||
| } | ||
| function idsIn(filePath) { | ||
| try { | ||
| const doc = yaml.load(fs.readFileSync(filePath, "utf8")); | ||
| if (!doc || !Array.isArray(doc.engrams)) return []; | ||
| return doc.engrams.map((e) => e?.id).filter((id) => typeof id === "string"); | ||
| } catch { | ||
| return []; | ||
| } | ||
| } | ||
| function idsCreatedAfter(root, since) { | ||
| const dir = path.join(root, "history"); | ||
| if (!fs.existsSync(dir)) return []; | ||
| const ids = []; | ||
| for (const name of fs.readdirSync(dir)) { | ||
| if (!name.endsWith(".jsonl")) continue; | ||
| let lines; | ||
| try { | ||
| lines = fs.readFileSync(path.join(dir, name), "utf8").split("\n"); | ||
| } catch { | ||
| continue; | ||
| } | ||
| for (const line of lines) { | ||
| if (!line.trim()) continue; | ||
| try { | ||
| const ev = JSON.parse(line); | ||
| if (typeof ev?.timestamp !== "string" || ev.timestamp <= since) continue; | ||
| if (typeof ev?.engram_id === "string") ids.push(ev.engram_id); | ||
| } catch { | ||
| } | ||
| } | ||
| } | ||
| return [...new Set(ids)]; | ||
| } | ||
| function restoreBackup(root, storePath, opts = {}) { | ||
| let plan; | ||
| const superseded = `${storePath}.superseded-${Date.now()}`; | ||
| withLock(storePath, () => { | ||
| plan = planRestore(root, storePath, opts.stamp); | ||
| if (!opts.force) { | ||
| const problems = []; | ||
| if (!plan.validity.ok) problems.push(...plan.validity.reasons); | ||
| if (!plan.integrityOk) { | ||
| problems.push( | ||
| plan.backup.sha256 === void 0 ? "no sha256 sidecar \u2014 cannot verify the backup is intact" : "sha256 does not match the sidecar \u2014 the backup itself is damaged" | ||
| ); | ||
| } | ||
| if (problems.length > 0) { | ||
| throw new Error( | ||
| `[plur] refusing to restore ${plan.backup.path}: ${problems.join("; ")}. | ||
| Restoring is a whole-corpus overwrite; doing it from a backup that does not verify would replace a damaged store with a differently damaged one. | ||
| Pass force to override if you have inspected the file yourself.` | ||
| ); | ||
| } | ||
| } | ||
| if (fs.existsSync(storePath)) { | ||
| fs.copyFileSync(storePath, superseded); | ||
| flushFileAt(superseded); | ||
| } | ||
| atomicWrite(storePath, fs.readFileSync(plan.backup.path, "utf8")); | ||
| }); | ||
| if (plan.wouldLose.length > 0) { | ||
| logger.warning( | ||
| `[plur:restore] ${plan.wouldLose.length} engram(s) present before the restore are not in this backup: ${plan.wouldLose.slice(0, 10).join(", ")}${plan.wouldLose.length > 10 ? ", \u2026" : ""}. The pre-restore store was kept at ${superseded}.` | ||
| ); | ||
| } | ||
| if (plan.unrecoverable.length > 0) { | ||
| logger.warning( | ||
| `[plur:restore] history records ${plan.unrecoverable.length} engram(s) created after this backup that it does not contain: ${plan.unrecoverable.slice(0, 10).join(", ")}${plan.unrecoverable.length > 10 ? ", \u2026" : ""}.` | ||
| ); | ||
| } | ||
| return { ...plan, restored: true, supersededPath: superseded }; | ||
| } | ||
| // src/history.ts | ||
| import * as fs2 from "fs"; | ||
| import { join as join2 } from "path"; | ||
| import { createHash as createHash2 } from "crypto"; | ||
| function appendHistory(root, event) { | ||
| const historyDir = join2(root, "history"); | ||
| if (!fs2.existsSync(historyDir)) { | ||
| fs2.mkdirSync(historyDir, { recursive: true }); | ||
| } | ||
| const date = event.timestamp.slice(0, 7); | ||
| const filePath = join2(historyDir, `${date}.jsonl`); | ||
| const line = JSON.stringify(event) + "\n"; | ||
| try { | ||
| const fd = fs2.openSync(filePath, "a"); | ||
| try { | ||
| fs2.writeSync(fd, line); | ||
| try { | ||
| fs2.fsyncSync(fd); | ||
| } catch { | ||
| } | ||
| } finally { | ||
| fs2.closeSync(fd); | ||
| } | ||
| } catch (err) { | ||
| if (!warnedHistoryPaths.has(filePath)) { | ||
| warnedHistoryPaths.add(filePath); | ||
| logger.warning( | ||
| `[plur] history could not be written to ${filePath}: ${err.message}. The operation itself succeeded. While this persists, \`plur restore\` cannot name unrecoverable engrams and engram-id allocation loses its cross-compaction guarantee (#816).` | ||
| ); | ||
| } | ||
| } | ||
| } | ||
| var warnedHistoryPaths = /* @__PURE__ */ new Set(); | ||
| function readHistory(root, yearMonth) { | ||
| const filePath = join2(root, "history", `${yearMonth}.jsonl`); | ||
| if (!fs2.existsSync(filePath)) return []; | ||
| const content = fs2.readFileSync(filePath, "utf8"); | ||
| const lines = content.split("\n").filter((l) => l.trim().length > 0); | ||
| const events = []; | ||
| for (const line of lines) { | ||
| try { | ||
| events.push(JSON.parse(line)); | ||
| } catch { | ||
| } | ||
| } | ||
| return events; | ||
| } | ||
| function mintedIdsWithPrefix(root, yearMonth, prefixes) { | ||
| try { | ||
| const out = []; | ||
| for (const ev of readHistory(root, yearMonth)) { | ||
| if (ev.event !== "engram_created") continue; | ||
| const id = ev.engram_id; | ||
| if (typeof id === "string" && prefixes.some((p) => id.startsWith(p))) out.push(id); | ||
| } | ||
| return out; | ||
| } catch { | ||
| return []; | ||
| } | ||
| } | ||
| function listHistoryMonths(root) { | ||
| const historyDir = join2(root, "history"); | ||
| if (!fs2.existsSync(historyDir)) return []; | ||
| return fs2.readdirSync(historyDir).filter((f) => f.endsWith(".jsonl")).map((f) => f.replace(".jsonl", "")).sort(); | ||
| } | ||
| function readHistoryForEngram(root, engramId) { | ||
| const months = listHistoryMonths(root); | ||
| const events = []; | ||
| for (const month of months) { | ||
| const monthEvents = readHistory(root, month); | ||
| for (const event of monthEvents) { | ||
| if (event.engram_id === engramId) { | ||
| events.push(event); | ||
| } | ||
| } | ||
| } | ||
| return events; | ||
| } | ||
| var _PROC_SALT = (process.pid % 1296).toString(36).padStart(2, "0"); | ||
| var _evtSeq = 0; | ||
| var _injSeq = 0; | ||
| function generateEventId() { | ||
| return `EVT-${Date.now()}-${_PROC_SALT}${(_evtSeq++).toString(36).padStart(4, "0")}`; | ||
| } | ||
| function generateInjectionId() { | ||
| return `INJ-${Date.now()}-${_PROC_SALT}${(_injSeq++).toString(36).padStart(4, "0")}`; | ||
| } | ||
| function computeQueryHash(task) { | ||
| const normalized = task.toLowerCase().replace(/\s+/g, " ").trim(); | ||
| return createHash2("sha256").update(normalized).digest("hex").slice(0, 16); | ||
| } | ||
| function findLatestInjectionFor(root, engramId, maxMonths = 2) { | ||
| const now = /* @__PURE__ */ new Date(); | ||
| const allowed = /* @__PURE__ */ new Set(); | ||
| for (let i = 0; i < maxMonths; i++) { | ||
| const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1)); | ||
| allowed.add(d.toISOString().slice(0, 7)); | ||
| } | ||
| const months = listHistoryMonths(root).filter((m) => allowed.has(m)).reverse(); | ||
| for (const month of months) { | ||
| let latest = null; | ||
| for (const event of readHistory(root, month)) { | ||
| if (event.event !== "co_injection") continue; | ||
| const ids = event.data.ids; | ||
| if (!Array.isArray(ids) || !ids.includes(engramId)) continue; | ||
| if (!latest || event.timestamp > latest.timestamp) latest = event; | ||
| } | ||
| if (latest) return { injection_id: latest.engram_id, timestamp: latest.timestamp }; | ||
| } | ||
| return null; | ||
| } | ||
| var INJECTION_SOURCES = /* @__PURE__ */ new Set([ | ||
| "session_start", | ||
| "inject", | ||
| "hook", | ||
| "unknown" | ||
| ]); | ||
| var ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/; | ||
| function readCoInjections(root, months) { | ||
| const events = []; | ||
| let skipped = 0; | ||
| const wanted = months ? new Set(months) : null; | ||
| for (const month of listHistoryMonths(root)) { | ||
| if (wanted && !wanted.has(month)) continue; | ||
| for (const event of readHistory(root, month)) { | ||
| if (event.event !== "co_injection") continue; | ||
| const raw = event.data; | ||
| if (!Array.isArray(raw.ids) || typeof raw.query_hash !== "string") { | ||
| skipped++; | ||
| continue; | ||
| } | ||
| if (typeof event.timestamp !== "string" || !ISO_TIMESTAMP.test(event.timestamp)) { | ||
| skipped++; | ||
| continue; | ||
| } | ||
| const ids = raw.ids.filter((id) => typeof id === "string" && id.length > 0); | ||
| if (ids.length !== raw.ids.length) skipped++; | ||
| const data = { ids, query_hash: raw.query_hash }; | ||
| if (typeof raw.tokens_used === "number" && Number.isFinite(raw.tokens_used)) { | ||
| data.tokens_used = raw.tokens_used; | ||
| } | ||
| if (raw.source !== void 0) { | ||
| data.source = INJECTION_SOURCES.has(raw.source) ? raw.source : "unknown"; | ||
| } | ||
| if (typeof raw.scope === "string") data.scope = raw.scope; | ||
| if (typeof raw.session_id === "string") data.session_id = raw.session_id; | ||
| events.push({ injection_id: event.engram_id, timestamp: event.timestamp, data }); | ||
| } | ||
| } | ||
| events.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); | ||
| return { events, skipped }; | ||
| } | ||
| function countInjectionEvents(root) { | ||
| const counts = { | ||
| co_injection: 0, | ||
| injection_outcome: 0, | ||
| outcome_positive: 0, | ||
| outcome_negative: 0 | ||
| }; | ||
| for (const month of listHistoryMonths(root)) { | ||
| for (const event of readHistory(root, month)) { | ||
| if (event.event === "co_injection") { | ||
| counts.co_injection++; | ||
| } else if (event.event === "injection_outcome") { | ||
| counts.injection_outcome++; | ||
| if (event.data.signal === "positive") counts.outcome_positive++; | ||
| else if (event.data.signal === "negative") counts.outcome_negative++; | ||
| } | ||
| } | ||
| } | ||
| return counts; | ||
| } | ||
| // src/content-hash.ts | ||
| import { createHash as createHash3 } from "crypto"; | ||
| var NON_WORD = /[^\p{L}\p{N}\p{M}_\s]/gu; | ||
| function normalizeStatement(statement) { | ||
| return statement.toLowerCase().replace(NON_WORD, "").replace(/\s+/g, " ").trim(); | ||
| } | ||
| function computeContentHash(statement) { | ||
| const normalized = normalizeStatement(statement); | ||
| return createHash3("sha256").update(normalized).digest("hex"); | ||
| } | ||
| function isHashable(statement) { | ||
| return normalizeStatement(statement).length > 0; | ||
| } | ||
| // src/dedup.ts | ||
| function buildDedupPrompt(newStatement, candidates) { | ||
| const candidateList = candidates.map( | ||
| (c, i) => `${i + 1}. [${c.id}] (${c.type}${c.domain ? ", domain: " + c.domain : ""}) | ||
| "${c.statement}"` | ||
| ).join("\n"); | ||
| return `You are a memory deduplication system. Compare a new memory statement against existing ones. | ||
| NEW STATEMENT: | ||
| "${newStatement}" | ||
| EXISTING ENGRAMS: | ||
| ${candidateList} | ||
| For each existing engram, answer: | ||
| 1. RELATIONSHIP: Is the new statement a DUPLICATE (same meaning), EVOLUTION (updated version of same knowledge), COMPLEMENTARY (related but different angle), or UNRELATED? | ||
| 2. RICHNESS: Does the new statement contain more specific, actionable information than the existing one? (yes/no) | ||
| Then give your OVERALL DECISION (exactly one): | ||
| - NOOP: New statement is an exact duplicate of an existing engram. Return the ID. | ||
| - UPDATE: New statement is an evolution with MORE information. Return the ID to update. | ||
| - MERGE: New statement and an existing one are complementary \u2014 combining them preserves both. Return the ID to merge with. | ||
| - ADD: New statement is genuinely new knowledge. | ||
| Respond in this exact format: | ||
| DECISION: <ADD|UPDATE|MERGE|NOOP> | ||
| TARGET: <engram ID if UPDATE/MERGE/NOOP, or "none" if ADD> | ||
| REASON: <one sentence explanation>`; | ||
| } | ||
| function buildBatchDedupPrompt(statements, existingEngrams) { | ||
| const stmtList = statements.map((s, i) => `${i + 1}. "${s}"`).join("\n"); | ||
| const engramList = existingEngrams.map( | ||
| (e, i) => `${i + 1}. [${e.id}] (${e.type}${e.domain ? ", domain: " + e.domain : ""}) | ||
| "${e.statement}"` | ||
| ).join("\n"); | ||
| return `You are a memory deduplication system. Compare NEW statements against existing engrams. | ||
| NEW STATEMENTS: | ||
| ${stmtList} | ||
| EXISTING ENGRAMS: | ||
| ${engramList} | ||
| For each NEW statement, decide: | ||
| - NOOP: Exact duplicate of an existing engram. | ||
| - UPDATE: Evolution with more info than existing. | ||
| - MERGE: Complementary with existing \u2014 combine. | ||
| - ADD: Genuinely new knowledge. | ||
| Respond with one block per new statement: | ||
| STATEMENT_1: | ||
| DECISION: <ADD|UPDATE|MERGE|NOOP> | ||
| TARGET: <engram ID or "none"> | ||
| STATEMENT_2: | ||
| ...`; | ||
| } | ||
| function parseDedupResponse(response) { | ||
| const decisionMatch = response.match(/DECISION:\s*(ADD|UPDATE|MERGE|NOOP)/i); | ||
| const targetMatch = response.match(/TARGET:\s*([^\n]+)/i); | ||
| const reasonMatch = response.match(/REASON:\s*([^\n]+)/i); | ||
| const decision = decisionMatch?.[1]?.toUpperCase() ?? "ADD"; | ||
| const targetRaw = targetMatch?.[1]?.trim() ?? "none"; | ||
| const target_id = targetRaw === "none" ? null : targetRaw.replace(/[^A-Za-z0-9-]/g, ""); | ||
| const reason = reasonMatch?.[1]?.trim() ?? ""; | ||
| return { decision, target_id, reason }; | ||
| } | ||
| export { | ||
| ExtractionProvenanceSchema, | ||
| getExtractionProvenance, | ||
| EngramSchemaPassthrough, | ||
| BACKUP_DIR, | ||
| validateStore, | ||
| maybeDailyBackup, | ||
| listBackups, | ||
| planRestore, | ||
| restoreBackup, | ||
| appendHistory, | ||
| readHistory, | ||
| mintedIdsWithPrefix, | ||
| listHistoryMonths, | ||
| readHistoryForEngram, | ||
| generateEventId, | ||
| generateInjectionId, | ||
| computeQueryHash, | ||
| findLatestInjectionFor, | ||
| readCoInjections, | ||
| countInjectionEvents, | ||
| normalizeStatement, | ||
| computeContentHash, | ||
| isHashable, | ||
| buildDedupPrompt, | ||
| buildBatchDedupPrompt, | ||
| parseDedupResponse | ||
| }; |
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/embedders/transformers-base.ts | ||
| var pipelineCache = /* @__PURE__ */ new Map(); | ||
| async function loadPipeline(modelId, dtype) { | ||
| const key = `${modelId}::${dtype ?? "fp32"}`; | ||
| let pending = pipelineCache.get(key); | ||
| if (!pending) { | ||
| pending = (async () => { | ||
| process.env.HF_HUB_DISABLE_XET ??= "1"; | ||
| const transformers = await import("@huggingface/transformers"); | ||
| const cacheDir = process.env.PLUR_MODEL_CACHE_DIR || process.env.HF_HOME; | ||
| if (cacheDir) { | ||
| ; | ||
| transformers.env.cacheDir = cacheDir; | ||
| } | ||
| return transformers.pipeline("feature-extraction", modelId, dtype ? { dtype } : void 0); | ||
| })(); | ||
| pipelineCache.set(key, pending); | ||
| } | ||
| return await pending; | ||
| } | ||
| function makeTransformersAdapter(config) { | ||
| const pooling = config.pooling; | ||
| const normalize = config.normalize ?? true; | ||
| async function embedOne(text) { | ||
| const pipe = await loadPipeline(config.modelId, config.dtype); | ||
| const result = await pipe(text, { pooling, normalize }); | ||
| const arr = result.data instanceof Float32Array ? result.data : new Float32Array(result.data); | ||
| if (arr.length !== config.dim) { | ||
| throw new Error( | ||
| `Embedder "${config.name}" returned ${arr.length}-dim vector, expected ${config.dim}` | ||
| ); | ||
| } | ||
| return arr; | ||
| } | ||
| return { | ||
| name: config.name, | ||
| dim: config.dim, | ||
| modelId: config.modelId, | ||
| embed: embedOne, | ||
| async embedBatch(texts) { | ||
| const out = []; | ||
| for (const t of texts) out.push(await embedOne(t)); | ||
| return out; | ||
| } | ||
| }; | ||
| } | ||
| // src/embedders/minilm.ts | ||
| var MINILM_MODEL_ID = "Xenova/all-MiniLM-L6-v2"; | ||
| function makeMiniLMAdapter() { | ||
| return makeTransformersAdapter({ | ||
| name: "minilm", | ||
| dim: 384, | ||
| modelId: MINILM_MODEL_ID, | ||
| pooling: "mean", | ||
| normalize: true, | ||
| dtype: "fp32" | ||
| }); | ||
| } | ||
| // src/embedders/bge-small.ts | ||
| var BGE_SMALL_MODEL_ID = "Xenova/bge-small-en-v1.5"; | ||
| function makeBgeSmallAdapter() { | ||
| return makeTransformersAdapter({ | ||
| name: "bge-small", | ||
| dim: 384, | ||
| modelId: BGE_SMALL_MODEL_ID, | ||
| pooling: "cls", | ||
| normalize: true, | ||
| dtype: "fp32" | ||
| }); | ||
| } | ||
| // src/embedders/bge-base.ts | ||
| var BGE_BASE_MODEL_ID = "Xenova/bge-base-en-v1.5"; | ||
| function makeBgeBaseAdapter() { | ||
| return makeTransformersAdapter({ | ||
| name: "bge-base", | ||
| dim: 768, | ||
| modelId: BGE_BASE_MODEL_ID, | ||
| pooling: "cls", | ||
| normalize: true, | ||
| dtype: "fp32" | ||
| }); | ||
| } | ||
| // src/embedders/embedding-gemma.ts | ||
| var EMBEDDING_GEMMA_MODEL_ID = "onnx-community/embeddinggemma-300m-ONNX"; | ||
| var DIM = 768; | ||
| var loaded = null; | ||
| async function load() { | ||
| if (!loaded) { | ||
| loaded = (async () => { | ||
| process.env.HF_HUB_DISABLE_XET ??= "1"; | ||
| const { AutoTokenizer, AutoModel } = await import("@huggingface/transformers"); | ||
| const tokenizer = await AutoTokenizer.from_pretrained(EMBEDDING_GEMMA_MODEL_ID); | ||
| const model = await AutoModel.from_pretrained(EMBEDDING_GEMMA_MODEL_ID, { dtype: "q8" }); | ||
| return { tokenizer, model }; | ||
| })(); | ||
| } | ||
| return loaded; | ||
| } | ||
| function makeEmbeddingGemmaAdapter() { | ||
| async function embedOne(text, role) { | ||
| const prefix = role === "query" ? "task: search result | query: " : "title: none | text: "; | ||
| const { tokenizer, model } = await load(); | ||
| const inputs = await tokenizer(prefix + text, { padding: true, truncation: true }); | ||
| const outputs = await model(inputs); | ||
| const raw = outputs.sentence_embedding.data; | ||
| const arr = raw instanceof Float32Array ? raw : new Float32Array(raw); | ||
| if (arr.length !== DIM) { | ||
| throw new Error(`EmbeddingGemma: expected ${DIM}-dim sentence_embedding, got ${arr.length}`); | ||
| } | ||
| return arr; | ||
| } | ||
| return { | ||
| // Suffix is a cache-space marker: embeddings.ts detects a name change and | ||
| // auto-invalidates any cached vectors built in a different space. '@graph' | ||
| // marked the switch off JS-side pooling; '-mcprefix' marks the #483 switch to | ||
| // the model-card role prefixes (old "query:"/"passage:" vectors are a | ||
| // different space and must be rebuilt). | ||
| name: "embedding-gemma@graph-mcprefix", | ||
| dim: DIM, | ||
| modelId: EMBEDDING_GEMMA_MODEL_ID, | ||
| embed: embedOne, | ||
| async embedBatch(texts) { | ||
| const out = []; | ||
| for (const t of texts) out.push(await embedOne(t)); | ||
| return out; | ||
| } | ||
| }; | ||
| } | ||
| // src/embedders/openai.ts | ||
| var OPENAI_3_LARGE_MODEL_ID = "text-embedding-3-large"; | ||
| var OPENAI_3_LARGE_DIM = 3072; | ||
| var OPENAI_3_LARGE_MAX_INPUT_TOKENS = 8191; | ||
| var ENDPOINT = "https://api.openai.com/v1/embeddings"; | ||
| var DEFAULT_MAX_BATCH_SIZE = 2048; | ||
| var DEFAULT_MAX_TOKENS_PER_REQUEST = 3e5; | ||
| var DEFAULT_TIMEOUT_MS = 3e4; | ||
| var DEFAULT_MAX_RETRIES = 3; | ||
| var DEFAULT_RETRY_BASE_MS = 1e3; | ||
| var MAX_RETRY_AFTER_MS = 6e4; | ||
| var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 503]); | ||
| function estimateTokens(text) { | ||
| return Math.ceil(text.length / 4); | ||
| } | ||
| function parseRetryAfterMs(header) { | ||
| if (!header) return null; | ||
| const secs = Number(header); | ||
| if (Number.isFinite(secs) && secs >= 0) { | ||
| return Math.min(secs * 1e3, MAX_RETRY_AFTER_MS); | ||
| } | ||
| const dateMs = Date.parse(header); | ||
| if (!Number.isNaN(dateMs)) { | ||
| return Math.min(Math.max(dateMs - Date.now(), 0), MAX_RETRY_AFTER_MS); | ||
| } | ||
| return null; | ||
| } | ||
| function readApiKey() { | ||
| const key = process.env.OPENAI_API_KEY; | ||
| if (!key || key.trim() === "") { | ||
| throw new Error( | ||
| "openai-3-large embedder requires OPENAI_API_KEY. Set the env var (export OPENAI_API_KEY=sk-...) or unset PLUR_EMBEDDER to fall back to the local default." | ||
| ); | ||
| } | ||
| return key; | ||
| } | ||
| function clampToTokenLimit(text, index) { | ||
| const maxChars = OPENAI_3_LARGE_MAX_INPUT_TOKENS * 4; | ||
| if (text.length <= maxChars) return text; | ||
| logger.warning( | ||
| `[openai-3-large] input ${index} is ~${estimateTokens(text)} tokens (limit ${OPENAI_3_LARGE_MAX_INPUT_TOKENS}) \u2014 truncating to avoid an HTTP 400 mid-batch` | ||
| ); | ||
| return text.slice(0, maxChars); | ||
| } | ||
| function parseVectors(json, expected) { | ||
| const data = json.data; | ||
| if (!Array.isArray(data) || data.length !== expected) { | ||
| throw new Error( | ||
| `openai-3-large embed returned ${data?.length ?? "no"} vectors for ${expected} inputs` | ||
| ); | ||
| } | ||
| return data.map((row, i) => { | ||
| const e = row.embedding; | ||
| if (!Array.isArray(e)) { | ||
| throw new Error(`openai-3-large returned non-array embedding at index ${i}`); | ||
| } | ||
| if (e.length !== OPENAI_3_LARGE_DIM) { | ||
| throw new Error( | ||
| `openai-3-large returned ${e.length}-dim vector at index ${i}, expected ${OPENAI_3_LARGE_DIM}` | ||
| ); | ||
| } | ||
| return Float32Array.from(e); | ||
| }); | ||
| } | ||
| async function postEmbed(texts, opts) { | ||
| const key = readApiKey(); | ||
| for (let attempt = 0; ; attempt++) { | ||
| const controller = new AbortController(); | ||
| const timer = setTimeout(() => controller.abort(), opts.timeoutMs); | ||
| let res; | ||
| try { | ||
| res = await opts.fetch(ENDPOINT, { | ||
| method: "POST", | ||
| headers: { | ||
| "content-type": "application/json", | ||
| authorization: `Bearer ${key}` | ||
| }, | ||
| body: JSON.stringify({ | ||
| model: OPENAI_3_LARGE_MODEL_ID, | ||
| input: texts | ||
| }), | ||
| signal: controller.signal | ||
| }); | ||
| } catch (err) { | ||
| if (controller.signal.aborted) { | ||
| throw new Error(`openai-3-large embed timed out after ${opts.timeoutMs}ms`); | ||
| } | ||
| throw err; | ||
| } finally { | ||
| clearTimeout(timer); | ||
| } | ||
| if (res.ok) { | ||
| return parseVectors(await res.json(), texts.length); | ||
| } | ||
| const body = await res.text().catch(() => ""); | ||
| if (RETRYABLE_STATUS.has(res.status) && attempt < opts.maxRetries) { | ||
| const delay = parseRetryAfterMs(res.headers.get("retry-after")) ?? opts.retryBaseMs * 2 ** attempt; | ||
| logger.warning( | ||
| `[openai-3-large] HTTP ${res.status} \u2014 retrying in ${delay}ms (attempt ${attempt + 1}/${opts.maxRetries})` | ||
| ); | ||
| await new Promise((r) => setTimeout(r, delay)); | ||
| continue; | ||
| } | ||
| throw new Error( | ||
| `openai-3-large embed failed: HTTP ${res.status} ${res.statusText}${body ? ` \u2014 ${body.slice(0, 200)}` : ""}` | ||
| ); | ||
| } | ||
| } | ||
| function chunkInputs(texts, opts) { | ||
| const chunks = []; | ||
| let current = []; | ||
| let currentTokens = 0; | ||
| for (const text of texts) { | ||
| const cost = estimateTokens(text); | ||
| if (current.length > 0 && (current.length >= opts.maxBatchSize || currentTokens + cost > opts.maxTokensPerRequest)) { | ||
| chunks.push(current); | ||
| current = []; | ||
| currentTokens = 0; | ||
| } | ||
| current.push(text); | ||
| currentTokens += cost; | ||
| } | ||
| if (current.length > 0) chunks.push(current); | ||
| return chunks; | ||
| } | ||
| function makeOpenAI3LargeAdapter(options) { | ||
| const opts = { | ||
| fetch: options?.fetch ?? ((...args) => globalThis.fetch(...args)), | ||
| timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, | ||
| maxRetries: options?.maxRetries ?? DEFAULT_MAX_RETRIES, | ||
| retryBaseMs: options?.retryBaseMs ?? DEFAULT_RETRY_BASE_MS, | ||
| maxBatchSize: options?.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE, | ||
| maxTokensPerRequest: options?.maxTokensPerRequest ?? DEFAULT_MAX_TOKENS_PER_REQUEST | ||
| }; | ||
| return { | ||
| name: "openai-3-large", | ||
| dim: OPENAI_3_LARGE_DIM, | ||
| modelId: OPENAI_3_LARGE_MODEL_ID, | ||
| async embed(text) { | ||
| const [v] = await postEmbed([clampToTokenLimit(text, 0)], opts); | ||
| return v; | ||
| }, | ||
| async embedBatch(texts) { | ||
| if (texts.length === 0) return []; | ||
| const clamped = texts.map((t, i) => clampToTokenLimit(t, i)); | ||
| const out = []; | ||
| for (const chunk of chunkInputs(clamped, opts)) { | ||
| out.push(...await postEmbed(chunk, opts)); | ||
| } | ||
| return out; | ||
| } | ||
| }; | ||
| } | ||
| // src/embedders/index.ts | ||
| var EMBEDDER_NAMES = [ | ||
| "minilm", | ||
| "bge-small", | ||
| "bge-base", | ||
| "embedding-gemma", | ||
| "openai-3-large" | ||
| ]; | ||
| var DEFAULT_EMBEDDER = "bge-small"; | ||
| var adapterCache = /* @__PURE__ */ new Map(); | ||
| function _resetEmbedderCache() { | ||
| adapterCache.clear(); | ||
| } | ||
| function getEmbedder(name) { | ||
| if (!EMBEDDER_NAMES.includes(name)) { | ||
| throw new Error(`Unknown embedder "${name}". Known: ${EMBEDDER_NAMES.join(", ")}`); | ||
| } | ||
| let adapter = adapterCache.get(name); | ||
| if (!adapter) { | ||
| adapter = build(name); | ||
| adapterCache.set(name, adapter); | ||
| } | ||
| return adapter; | ||
| } | ||
| function build(name) { | ||
| switch (name) { | ||
| case "minilm": | ||
| return makeMiniLMAdapter(); | ||
| case "bge-small": | ||
| return makeBgeSmallAdapter(); | ||
| case "bge-base": | ||
| return makeBgeBaseAdapter(); | ||
| case "embedding-gemma": | ||
| return makeEmbeddingGemmaAdapter(); | ||
| case "openai-3-large": | ||
| return makeOpenAI3LargeAdapter(); | ||
| default: { | ||
| const _exhaustive = name; | ||
| throw new Error(`Unhandled embedder name: ${String(_exhaustive)}`); | ||
| } | ||
| } | ||
| } | ||
| var warnedUnknown = false; | ||
| function resolveEmbedderName(env = process.env) { | ||
| const raw = env.PLUR_EMBEDDER?.trim(); | ||
| if (!raw) return DEFAULT_EMBEDDER; | ||
| if (EMBEDDER_NAMES.includes(raw)) return raw; | ||
| if (!warnedUnknown) { | ||
| logger.warning( | ||
| `[embedders] PLUR_EMBEDDER="${raw}" not recognised. Falling back to "${DEFAULT_EMBEDDER}". Known: ${EMBEDDER_NAMES.join(", ")}` | ||
| ); | ||
| warnedUnknown = true; | ||
| } | ||
| return DEFAULT_EMBEDDER; | ||
| } | ||
| function _resetResolveWarnings() { | ||
| warnedUnknown = false; | ||
| } | ||
| export { | ||
| EMBEDDER_NAMES, | ||
| DEFAULT_EMBEDDER, | ||
| _resetEmbedderCache, | ||
| getEmbedder, | ||
| resolveEmbedderName, | ||
| _resetResolveWarnings | ||
| }; |
| import { | ||
| atomicWrite | ||
| } from "./chunk-TXHLQGN3.js"; | ||
| import { | ||
| engramSearchText | ||
| } from "./chunk-2UD5K4YD.js"; | ||
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/embeddings.ts | ||
| import { existsSync, readFileSync, mkdirSync } from "fs"; | ||
| import { join, dirname } from "path"; | ||
| import { createHash } from "crypto"; | ||
| var EMBED_DIM = 384; | ||
| var embedPipeline = null; | ||
| var lastLoadError = null; | ||
| var transformersUnavailable = false; | ||
| function readDisabledFromEnv(env) { | ||
| const raw = env.PLUR_DISABLE_EMBEDDINGS; | ||
| if (!raw) return null; | ||
| const normalized = raw.trim().toLowerCase(); | ||
| if (normalized === "1" || normalized === "true" || normalized === "yes") { | ||
| return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var"; | ||
| } | ||
| return null; | ||
| } | ||
| var ENV_DISABLED_REASON = readDisabledFromEnv(process.env); | ||
| var embeddingsDisabled = ENV_DISABLED_REASON !== null; | ||
| var disabledReason = ENV_DISABLED_REASON; | ||
| function embedderStatus() { | ||
| return { | ||
| available: !embeddingsDisabled && !transformersUnavailable, | ||
| loaded: embedPipeline !== null, | ||
| lastError: lastLoadError, | ||
| disabled: embeddingsDisabled, | ||
| disabledReason | ||
| }; | ||
| } | ||
| function setEmbeddingsEnabled(enabled, reason) { | ||
| embeddingsDisabled = !enabled; | ||
| disabledReason = enabled ? null : reason ?? "embeddings disabled by config"; | ||
| if (!enabled) { | ||
| embedPipeline = null; | ||
| } | ||
| } | ||
| function resetEmbedder() { | ||
| transformersUnavailable = false; | ||
| lastLoadError = null; | ||
| embedPipeline = null; | ||
| } | ||
| function _setCachedEmbedder(adapter) { | ||
| embedPipeline = adapter; | ||
| transformersUnavailable = false; | ||
| lastLoadError = null; | ||
| } | ||
| async function getEmbedder() { | ||
| if (embeddingsDisabled) return null; | ||
| if (embedPipeline) return embedPipeline; | ||
| try { | ||
| const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-PJW3I32N.js"); | ||
| const adapter = getAdapter(resolveEmbedderName()); | ||
| embedPipeline = adapter; | ||
| transformersUnavailable = false; | ||
| lastLoadError = null; | ||
| return embedPipeline; | ||
| } catch (err) { | ||
| transformersUnavailable = true; | ||
| lastLoadError = err instanceof Error ? err.message : String(err); | ||
| return null; | ||
| } | ||
| } | ||
| async function embed(text, role) { | ||
| const embedder = await getEmbedder(); | ||
| if (!embedder) return null; | ||
| if (typeof embedder.embed === "function") { | ||
| let vector; | ||
| try { | ||
| vector = await embedder.embed(text, role); | ||
| } catch (err) { | ||
| transformersUnavailable = true; | ||
| lastLoadError = err instanceof Error ? err.message : String(err); | ||
| embedPipeline = null; | ||
| return null; | ||
| } | ||
| if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) { | ||
| throw new Error( | ||
| `Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.` | ||
| ); | ||
| } | ||
| return vector; | ||
| } | ||
| const result = await embedder(text, { pooling: "cls", normalize: true }); | ||
| return new Float32Array(result.data); | ||
| } | ||
| async function getActiveEmbedderMeta() { | ||
| const embedder = await getEmbedder(); | ||
| if (!embedder) return null; | ||
| if (typeof embedder.name === "string" && typeof embedder.dim === "number") { | ||
| return { name: embedder.name, dim: embedder.dim }; | ||
| } | ||
| return { name: "legacy-pipeline", dim: 0 }; | ||
| } | ||
| async function activeEmbedderDim() { | ||
| const meta = await getActiveEmbedderMeta(); | ||
| return meta && meta.dim > 0 ? meta.dim : null; | ||
| } | ||
| function cosineSimilarity(a, b) { | ||
| let dot = 0; | ||
| for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; | ||
| return dot; | ||
| } | ||
| var CACHE_VERSION = 1; | ||
| function emptyCache(meta) { | ||
| return { | ||
| meta: { | ||
| embedder_name: meta.name, | ||
| embedder_dim: meta.dim, | ||
| version: CACHE_VERSION | ||
| }, | ||
| entries: {} | ||
| }; | ||
| } | ||
| function loadCache(cachePath, active) { | ||
| if (!existsSync(cachePath)) return emptyCache(active); | ||
| try { | ||
| const raw = JSON.parse(readFileSync(cachePath, "utf8")); | ||
| if (!raw || typeof raw !== "object" || !raw.meta) { | ||
| logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`); | ||
| return emptyCache(active); | ||
| } | ||
| const meta = raw.meta; | ||
| if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) { | ||
| logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`); | ||
| return emptyCache(active); | ||
| } | ||
| const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {}; | ||
| return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries }; | ||
| } catch { | ||
| return emptyCache(active); | ||
| } | ||
| } | ||
| function saveCache(cachePath, cache) { | ||
| const dir = dirname(cachePath); | ||
| if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); | ||
| atomicWrite(cachePath, JSON.stringify(cache), { durable: false }); | ||
| } | ||
| function hashStatement(statement) { | ||
| return createHash("sha256").update(statement).digest("hex").slice(0, 16); | ||
| } | ||
| async function embeddingSearch(engrams, query, limit, storagePath) { | ||
| if (engrams.length === 0) return []; | ||
| const activeMeta = await getActiveEmbedderMeta(); | ||
| if (!activeMeta) return []; | ||
| const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json"; | ||
| const cache = loadCache(cachePath, activeMeta); | ||
| const queryEmbedding = await embed(query, "query"); | ||
| if (!queryEmbedding) { | ||
| return []; | ||
| } | ||
| const similarities = []; | ||
| for (const engram of engrams) { | ||
| const searchText = engramSearchText(engram); | ||
| const hash = hashStatement(searchText); | ||
| let engramEmbedding; | ||
| if (cache.entries[engram.id]?.hash === hash) { | ||
| engramEmbedding = new Float32Array(cache.entries[engram.id].embedding); | ||
| } else { | ||
| const emb = await embed(searchText); | ||
| if (!emb) return []; | ||
| engramEmbedding = emb; | ||
| cache.entries[engram.id] = { | ||
| hash, | ||
| embedding: Array.from(engramEmbedding) | ||
| }; | ||
| } | ||
| const score = cosineSimilarity(queryEmbedding, engramEmbedding); | ||
| similarities.push({ engram, score }); | ||
| } | ||
| saveCache(cachePath, cache); | ||
| similarities.sort((a, b) => b.score - a.score); | ||
| return similarities.slice(0, limit).map((s) => s.engram); | ||
| } | ||
| async function embeddingSearchWithScores(engrams, query, limit, storagePath) { | ||
| if (engrams.length === 0) return []; | ||
| const activeMeta = await getActiveEmbedderMeta(); | ||
| if (!activeMeta) return []; | ||
| const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json"; | ||
| const cache = loadCache(cachePath, activeMeta); | ||
| const queryEmbedding = await embed(query, "query"); | ||
| if (!queryEmbedding) { | ||
| return []; | ||
| } | ||
| const similarities = []; | ||
| for (const engram of engrams) { | ||
| const searchText = engramSearchText(engram); | ||
| const hash = hashStatement(searchText); | ||
| let engramEmbedding; | ||
| if (cache.entries[engram.id]?.hash === hash) { | ||
| engramEmbedding = new Float32Array(cache.entries[engram.id].embedding); | ||
| } else { | ||
| const emb = await embed(searchText); | ||
| if (!emb) return []; | ||
| engramEmbedding = emb; | ||
| cache.entries[engram.id] = { | ||
| hash, | ||
| embedding: Array.from(engramEmbedding) | ||
| }; | ||
| } | ||
| const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding); | ||
| const score = Math.max(0, Math.min(1, rawScore)); | ||
| similarities.push({ engram, score }); | ||
| } | ||
| saveCache(cachePath, cache); | ||
| similarities.sort((a, b) => b.score - a.score); | ||
| return similarities.slice(0, limit); | ||
| } | ||
| async function rebuildJsonCache(engrams, storagePath, opts) { | ||
| const activeMeta = await getActiveEmbedderMeta(); | ||
| if (!activeMeta) { | ||
| return { reembedded: 0, skipped: true, reason: "embedder unavailable" }; | ||
| } | ||
| const cachePath = join(storagePath, ".embeddings-cache.json"); | ||
| const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta); | ||
| let count = 0; | ||
| for (const engram of engrams) { | ||
| const searchText = engramSearchText(engram); | ||
| const hash = hashStatement(searchText); | ||
| if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue; | ||
| const vec = await embed(searchText); | ||
| if (!vec) { | ||
| return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" }; | ||
| } | ||
| cache.entries[engram.id] = { hash, embedding: Array.from(vec) }; | ||
| count++; | ||
| } | ||
| saveCache(cachePath, cache); | ||
| return { reembedded: count, skipped: false }; | ||
| } | ||
| export { | ||
| EMBED_DIM, | ||
| readDisabledFromEnv, | ||
| embedderStatus, | ||
| setEmbeddingsEnabled, | ||
| resetEmbedder, | ||
| _setCachedEmbedder, | ||
| embed, | ||
| activeEmbedderDim, | ||
| cosineSimilarity, | ||
| embeddingSearch, | ||
| embeddingSearchWithScores, | ||
| rebuildJsonCache | ||
| }; |
| import { | ||
| DEFAULT_EMBEDDER, | ||
| EMBEDDER_NAMES, | ||
| _resetEmbedderCache, | ||
| _resetResolveWarnings, | ||
| getEmbedder, | ||
| resolveEmbedderName | ||
| } from "./chunk-VNFYIPQL.js"; | ||
| import "./chunk-E4YVUWMJ.js"; | ||
| export { | ||
| DEFAULT_EMBEDDER, | ||
| EMBEDDER_NAMES, | ||
| _resetEmbedderCache, | ||
| _resetResolveWarnings, | ||
| getEmbedder, | ||
| resolveEmbedderName | ||
| }; |
| import { | ||
| EMBED_DIM, | ||
| _setCachedEmbedder, | ||
| activeEmbedderDim, | ||
| cosineSimilarity, | ||
| embed, | ||
| embedderStatus, | ||
| embeddingSearch, | ||
| embeddingSearchWithScores, | ||
| readDisabledFromEnv, | ||
| rebuildJsonCache, | ||
| resetEmbedder, | ||
| setEmbeddingsEnabled | ||
| } from "./chunk-W55ICNED.js"; | ||
| import "./chunk-TXHLQGN3.js"; | ||
| import "./chunk-2UD5K4YD.js"; | ||
| import "./chunk-E4YVUWMJ.js"; | ||
| export { | ||
| EMBED_DIM, | ||
| _setCachedEmbedder, | ||
| activeEmbedderDim, | ||
| cosineSimilarity, | ||
| embed, | ||
| embedderStatus, | ||
| embeddingSearch, | ||
| embeddingSearchWithScores, | ||
| readDisabledFromEnv, | ||
| rebuildJsonCache, | ||
| resetEmbedder, | ||
| setEmbeddingsEnabled | ||
| }; |
| import { | ||
| MAX_SPACELESS_RUN_CHARS, | ||
| MIN_TOKEN_LENGTH, | ||
| TOKENIZER_VERSION, | ||
| computeIdf, | ||
| embeddingContentHash, | ||
| engramSearchText, | ||
| extendCorpusStats, | ||
| ftsScore, | ||
| ftsTokenize, | ||
| hashEmbeddedText, | ||
| searchEngrams, | ||
| searchTextFrom, | ||
| termMatches | ||
| } from "./chunk-2UD5K4YD.js"; | ||
| export { | ||
| MAX_SPACELESS_RUN_CHARS, | ||
| MIN_TOKEN_LENGTH, | ||
| TOKENIZER_VERSION, | ||
| computeIdf, | ||
| embeddingContentHash, | ||
| engramSearchText, | ||
| extendCorpusStats, | ||
| ftsScore, | ||
| ftsTokenize, | ||
| hashEmbeddedText, | ||
| searchEngrams, | ||
| searchTextFrom, | ||
| termMatches | ||
| }; |
| import { | ||
| appendHistory, | ||
| buildDedupPrompt, | ||
| computeContentHash, | ||
| maybeDailyBackup, | ||
| parseDedupResponse | ||
| } from "./chunk-EJQ4MGQV.js"; | ||
| import { | ||
| withAsyncLock | ||
| } from "./chunk-TXHLQGN3.js"; | ||
| import { | ||
| searchTextFrom | ||
| } from "./chunk-2UD5K4YD.js"; | ||
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/learn-async.ts | ||
| var NEAR_DUPLICATE_OBSERVATION_FLOOR = 0.75; | ||
| async function persistOne(deps, corpus, changed) { | ||
| if (deps.store.updateMany) { | ||
| await deps.store.updateMany([changed]); | ||
| deps.store.invalidate(); | ||
| return; | ||
| } | ||
| await deps.store.save(corpus); | ||
| } | ||
| async function withStoreLock(deps, fn) { | ||
| const guarded = async () => { | ||
| try { | ||
| maybeDailyBackup(deps.rootPath, deps.engramsPath); | ||
| } catch { | ||
| } | ||
| return await fn(); | ||
| }; | ||
| if (deps.store.withExclusiveAccess) return await deps.store.withExclusiveAccess(guarded); | ||
| return await withAsyncLock(deps.engramsPath, guarded); | ||
| } | ||
| function demoteIfSensitive(deps, engram, newStatement) { | ||
| const tags = Array.isArray(engram.tags) ? engram.tags.filter((t) => typeof t === "string") : []; | ||
| const scanText = tags.length ? `${newStatement} | ||
| ${tags.join(" ")}` : newStatement; | ||
| const offending = deps.offendingHitsForScope(scanText, engram.scope ?? "global"); | ||
| if (offending.length === 0) return; | ||
| const patterns = [...new Set(offending.map((h) => h.pattern))].join(", "); | ||
| logger.warning( | ||
| `[plur] sensitive content (${patterns}) held back from shared scope "${engram.scope}" \u2014 demoted to local/private so it is not written to a shared store. Re-scope deliberately if this is a false positive.` | ||
| ); | ||
| const from = engram.scope ?? "global"; | ||
| engram.scope = "local"; | ||
| engram.visibility = "private"; | ||
| engram.structured_data = { | ||
| ...engram.structured_data ?? {}, | ||
| _demoted: { from, to: "local", patterns } | ||
| }; | ||
| } | ||
| async function executeDedupDecision(deps, statement, context, decision, targetId) { | ||
| switch (decision) { | ||
| case "NOOP": { | ||
| if (targetId) { | ||
| const existing = await deps.getById(targetId); | ||
| if (existing) return { engram: existing, decision: "NOOP", existing_id: targetId }; | ||
| } | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| case "UPDATE": { | ||
| if (targetId) { | ||
| const existing = await deps.getById(targetId); | ||
| if (existing && existing.commitment !== "locked") { | ||
| const result = await withStoreLock(deps, async () => { | ||
| const engrams = await deps.store.load(); | ||
| const idx = engrams.findIndex((e) => e.id === targetId); | ||
| if (idx === -1) return null; | ||
| const updated = { ...engrams[idx] }; | ||
| updated.statement = statement; | ||
| updated.content_hash = computeContentHash(statement); | ||
| updated.engram_version = (updated.engram_version ?? 1) + 1; | ||
| updated.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10); | ||
| if (context?.tags) updated.tags = [.../* @__PURE__ */ new Set([...updated.tags, ...context.tags])]; | ||
| demoteIfSensitive(deps, updated, updated.statement); | ||
| engrams[idx] = updated; | ||
| await persistOne(deps, engrams, updated); | ||
| await deps.syncIndex(); | ||
| appendHistory(deps.rootPath, { | ||
| event: "engram_updated", | ||
| engram_id: targetId, | ||
| timestamp: (/* @__PURE__ */ new Date()).toISOString(), | ||
| data: { old_statement: existing.statement, new_statement: statement, reason: "LLM dedup UPDATE" } | ||
| }); | ||
| return { engram: updated, decision: "UPDATE", existing_id: targetId }; | ||
| }); | ||
| if (result) return result; | ||
| } | ||
| } | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| case "MERGE": { | ||
| if (targetId) { | ||
| const existing = await deps.getById(targetId); | ||
| if (existing && existing.commitment !== "locked") { | ||
| const result = await withStoreLock(deps, async () => { | ||
| const engrams = await deps.store.load(); | ||
| const idx = engrams.findIndex((e) => e.id === targetId); | ||
| if (idx === -1) return null; | ||
| const merged = { ...engrams[idx] }; | ||
| merged.statement = `${merged.statement} ${statement}`; | ||
| merged.content_hash = computeContentHash(merged.statement); | ||
| merged.engram_version = (merged.engram_version ?? 1) + 1; | ||
| merged.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10); | ||
| if (context?.tags) merged.tags = [.../* @__PURE__ */ new Set([...merged.tags, ...context.tags])]; | ||
| if (0.7 > merged.activation.retrieval_strength) merged.activation.retrieval_strength = 0.7; | ||
| demoteIfSensitive(deps, merged, merged.statement); | ||
| engrams[idx] = merged; | ||
| await persistOne(deps, engrams, merged); | ||
| await deps.syncIndex(); | ||
| appendHistory(deps.rootPath, { | ||
| event: "engram_merged", | ||
| engram_id: targetId, | ||
| timestamp: (/* @__PURE__ */ new Date()).toISOString(), | ||
| data: { merged_statement: statement, reason: "LLM dedup MERGE" } | ||
| }); | ||
| return { engram: merged, decision: "MERGE", existing_id: targetId }; | ||
| }); | ||
| if (result) return result; | ||
| } | ||
| } | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| case "ADD": | ||
| default: | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| } | ||
| async function learnAsync(deps, statement, context) { | ||
| const hashMatch = await deps.hashDedup(statement, context?.scope); | ||
| if (hashMatch) { | ||
| return { engram: hashMatch, decision: "NOOP", existing_id: hashMatch.id }; | ||
| } | ||
| const { enabled = true, threshold = 0.85, mode = "llm" } = deps.dedupConfig; | ||
| if (!enabled || mode === "off") { | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| let candidates = []; | ||
| try { | ||
| candidates = await deps.recallHybrid(statement, { limit: 5 }); | ||
| } catch { | ||
| candidates = await deps.recall(statement, { limit: 5 }); | ||
| } | ||
| if (candidates.length === 0) { | ||
| candidates = await deps.recall(statement, { limit: 5 }); | ||
| } | ||
| candidates = candidates.filter((c) => c.status === "active"); | ||
| if (context?.scope) { | ||
| candidates = candidates.filter((c) => c.scope === context.scope); | ||
| } | ||
| if (candidates.length === 0) { | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| const llm = context?.llm; | ||
| let decision = "ADD"; | ||
| let targetId = null; | ||
| let dedupMode = "hash-only"; | ||
| let nearDuplicates; | ||
| if (mode === "llm" && llm && deps.isLlmAvailable()) { | ||
| dedupMode = "llm"; | ||
| try { | ||
| const prompt = buildDedupPrompt( | ||
| statement, | ||
| candidates.map((c) => ({ id: c.id, statement: c.statement, type: c.type, domain: c.domain })) | ||
| ); | ||
| const response = await llm(prompt); | ||
| const parsed = parseDedupResponse(response); | ||
| decision = parsed.decision; | ||
| targetId = parsed.target_id; | ||
| deps.recordLlmSuccess(); | ||
| } catch (err) { | ||
| logger.warning(`LLM dedup failed, falling back to local similarity: ${err}`); | ||
| deps.recordLlmFailure(); | ||
| decision = "ADD"; | ||
| dedupMode = "hash-only"; | ||
| } | ||
| } | ||
| if (dedupMode !== "llm" && deps.similarityScores) { | ||
| try { | ||
| const query = searchTextFrom({ | ||
| statement, | ||
| domain: context?.domain, | ||
| tags: context?.tags, | ||
| rationale: context?.rationale, | ||
| source: context?.source, | ||
| dual_coding: context?.dual_coding, | ||
| knowledge_anchors: context?.knowledge_anchors | ||
| }); | ||
| const scores = (await deps.similarityScores(query, candidates)).slice().sort((a, b) => b.score - a.score); | ||
| if (scores.length > 0) { | ||
| dedupMode = "cosine"; | ||
| nearDuplicates = scores.slice(0, 3); | ||
| const top = scores[0]; | ||
| if (top.score >= NEAR_DUPLICATE_OBSERVATION_FLOOR) { | ||
| try { | ||
| appendHistory(deps.rootPath, { | ||
| event: "dedup_near_duplicate", | ||
| engram_id: top.id, | ||
| timestamp: (/* @__PURE__ */ new Date()).toISOString(), | ||
| data: { | ||
| statement: statement.slice(0, 200), | ||
| top_score: Number(top.score.toFixed(4)), | ||
| reporting_threshold: threshold, | ||
| above_reporting_threshold: top.score >= threshold, | ||
| scope: context?.scope ?? null, | ||
| // The asymmetry that broke the bar. Recorded per write so the | ||
| // distribution can be split by it instead of averaging over it. | ||
| incoming_has_domain: Boolean(context?.domain), | ||
| incoming_has_rationale: Boolean(context?.rationale) | ||
| } | ||
| }); | ||
| } catch (err) { | ||
| logger.warning(`could not record near-duplicate observation: ${err}`); | ||
| } | ||
| } | ||
| } | ||
| } catch (err) { | ||
| logger.warning(`local similarity dedup unavailable, adding without it: ${err}`); | ||
| dedupMode = "hash-only"; | ||
| nearDuplicates = void 0; | ||
| } | ||
| } | ||
| const executed = await executeDedupDecision(deps, statement, context, decision, targetId); | ||
| return { ...executed, dedup: { mode: dedupMode, ...nearDuplicates ? { near_duplicates: nearDuplicates } : {} } }; | ||
| } | ||
| async function learnBatch(deps, statements, llm, opts = {}) { | ||
| const results = []; | ||
| const failures = []; | ||
| const stats = { added: 0, updated: 0, merged: 0, noops: 0, failed: 0 }; | ||
| const maxLlmCalls = opts.maxLlmCalls ?? 50; | ||
| let llmCallsUsed = 0; | ||
| let capWarned = false; | ||
| for (let i = 0; i < statements.length; i++) { | ||
| const { statement, context } = statements[i]; | ||
| const stmtLlm = context?.llm ?? llm; | ||
| let effectiveLlm = stmtLlm; | ||
| if (stmtLlm) { | ||
| if (llmCallsUsed >= maxLlmCalls) { | ||
| effectiveLlm = void 0; | ||
| if (!capWarned) { | ||
| logger.warning(`learnBatch: maxLlmCalls (${maxLlmCalls}) reached \u2014 remaining statements fall back to local cosine dedup (see each result's dedup.mode)`); | ||
| capWarned = true; | ||
| } | ||
| } else { | ||
| effectiveLlm = async (prompt) => { | ||
| llmCallsUsed++; | ||
| return stmtLlm(prompt); | ||
| }; | ||
| } | ||
| } | ||
| const ctx = { ...context, llm: effectiveLlm }; | ||
| try { | ||
| const result = await learnAsync(deps, statement, ctx); | ||
| results.push({ ...result, input_index: i }); | ||
| const key = result.decision.toLowerCase(); | ||
| if (key === "noop") stats.noops++; | ||
| else if (key === "update") stats.updated++; | ||
| else if (key === "merge") stats.merged++; | ||
| else stats.added++; | ||
| } catch (err) { | ||
| stats.failed++; | ||
| failures.push({ index: i, statement, error: err instanceof Error ? err.message : String(err) }); | ||
| logger.warning(`learnBatch: statement ${i} failed \u2014 ${err instanceof Error ? err.message : String(err)}`); | ||
| } | ||
| } | ||
| return { results, stats, failures }; | ||
| } | ||
| export { | ||
| NEAR_DUPLICATE_OBSERVATION_FLOOR, | ||
| learnAsync, | ||
| learnBatch | ||
| }; |
+7
-4
| { | ||
| "name": "@plur-ai/core", | ||
| "version": "0.17.2", | ||
| "version": "0.18.0", | ||
| "type": "module", | ||
@@ -12,3 +12,3 @@ "main": "dist/index.js", | ||
| "@electric-sql/pglite": "^0.4.6", | ||
| "js-yaml": "^4.3.0", | ||
| "js-yaml": "^4.3.1", | ||
| "zod": "^3.23.0" | ||
@@ -18,7 +18,7 @@ }, | ||
| "@huggingface/transformers": "^3.8.1", | ||
| "better-sqlite3": "^11.0.0", | ||
| "better-sqlite3": "^12.0.0", | ||
| "pg": "^8.16.3" | ||
| }, | ||
| "devDependencies": { | ||
| "@types/better-sqlite3": "^7.6.0", | ||
| "@types/better-sqlite3": "^9.0.0", | ||
| "@types/js-yaml": "^4.0.0", | ||
@@ -30,2 +30,5 @@ "@types/pg": "^8.15.6", | ||
| }, | ||
| "engines": { | ||
| "node": ">=20" | ||
| }, | ||
| "license": "Apache-2.0", | ||
@@ -32,0 +35,0 @@ "description": "Shared memory engine for AI agents — learn, recall, forget, feedback", |
| // src/fts.ts | ||
| import { createHash } from "crypto"; | ||
| var STOP_WORDS = /* @__PURE__ */ new Set([ | ||
| "the", | ||
| "and", | ||
| "for", | ||
| "that", | ||
| "this", | ||
| "with", | ||
| "from", | ||
| "are", | ||
| "was", | ||
| "were", | ||
| "been", | ||
| "have", | ||
| "has", | ||
| "not", | ||
| "but", | ||
| "its", | ||
| "you", | ||
| "your", | ||
| "can", | ||
| "will", | ||
| "should", | ||
| "would", | ||
| "could", | ||
| "may", | ||
| "might" | ||
| ]); | ||
| var MIN_TOKEN_LENGTH = 2; | ||
| var TOKENIZER_VERSION = 2; | ||
| function ftsTokenize(text) { | ||
| const lower = text.toLowerCase(); | ||
| const tokens = lower.replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w)); | ||
| for (const run of lower.match(new RegExp("\\p{Script=Han}{2,}", "gu")) ?? []) { | ||
| for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2)); | ||
| } | ||
| return tokens; | ||
| } | ||
| function engramSearchText(engram) { | ||
| const parts = [engram.statement]; | ||
| if (engram.domain) parts.push(engram.domain.replace(/\./g, " ")); | ||
| if (engram.tags.length > 0) parts.push(engram.tags.join(" ")); | ||
| if (engram.entities) { | ||
| for (const e of engram.entities) { | ||
| parts.push(e.name); | ||
| if (e.type !== "other") parts.push(e.type); | ||
| } | ||
| } | ||
| if (engram.temporal) { | ||
| if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from); | ||
| if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until); | ||
| } | ||
| if (engram.rationale) parts.push(engram.rationale); | ||
| if (engram.source) parts.push(engram.source); | ||
| if (engram.dual_coding) { | ||
| if (engram.dual_coding.example) parts.push(engram.dual_coding.example); | ||
| if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy); | ||
| } | ||
| if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) { | ||
| for (const a of engram.knowledge_anchors) { | ||
| if (a.snippet) parts.push(a.snippet); | ||
| } | ||
| } | ||
| return parts.join(" "); | ||
| } | ||
| function embeddingContentHash(engram) { | ||
| return hashEmbeddedText(engramSearchText(engram)); | ||
| } | ||
| function hashEmbeddedText(text) { | ||
| return createHash("md5").update(text).digest("hex"); | ||
| } | ||
| function termMatches(t, qt) { | ||
| return t.includes(qt) || qt.startsWith(t); | ||
| } | ||
| function computeIdf(engrams, queryTokens, stats) { | ||
| if (stats) { | ||
| if (stats.N === 0) return /* @__PURE__ */ new Map(); | ||
| const idf2 = /* @__PURE__ */ new Map(); | ||
| for (const qt of queryTokens) { | ||
| const df = stats.df.get(qt) ?? 0; | ||
| idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df)))); | ||
| } | ||
| return idf2; | ||
| } | ||
| const N = engrams.length; | ||
| if (N === 0) return /* @__PURE__ */ new Map(); | ||
| const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e)))); | ||
| const idf = /* @__PURE__ */ new Map(); | ||
| for (const qt of queryTokens) { | ||
| let df = 0; | ||
| for (const termSet of engramTermSets) { | ||
| if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) { | ||
| df++; | ||
| } | ||
| } | ||
| idf.set(qt, Math.max(0, Math.log(N / (1 + df)))); | ||
| } | ||
| return idf; | ||
| } | ||
| function extendCorpusStats(stats, queryTokens, outsiders) { | ||
| if (outsiders.length === 0) return stats; | ||
| const termSets = []; | ||
| let totalLen = 0; | ||
| for (const e of outsiders) { | ||
| const terms = ftsTokenize(engramSearchText(e)); | ||
| totalLen += terms.length; | ||
| termSets.push(new Set(terms)); | ||
| } | ||
| const df = new Map(stats.df); | ||
| for (const qt of queryTokens) { | ||
| let added = 0; | ||
| for (const set of termSets) { | ||
| if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++; | ||
| } | ||
| if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added); | ||
| } | ||
| const N = stats.N + outsiders.length; | ||
| return { | ||
| N, | ||
| df, | ||
| avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0 | ||
| }; | ||
| } | ||
| var BM25_K1 = 1.2; | ||
| var BM25_B = 0.75; | ||
| function ftsScore(engram, queryTokens, idfWeights, avgDocLength) { | ||
| const allTerms = ftsTokenize(engramSearchText(engram)); | ||
| if (queryTokens.length === 0) return 0; | ||
| const docLen = allTerms.length; | ||
| const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen; | ||
| const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0); | ||
| let score = 0; | ||
| for (const qt of queryTokens) { | ||
| let effectiveIdf; | ||
| if (!idfWeights) { | ||
| effectiveIdf = 1; | ||
| } else if (hasNonZeroIdf) { | ||
| effectiveIdf = idfWeights.get(qt) ?? 0; | ||
| if (effectiveIdf === 0) continue; | ||
| } else { | ||
| effectiveIdf = 1; | ||
| } | ||
| let tf = 0; | ||
| for (const t of allTerms) { | ||
| if (termMatches(t, qt)) tf++; | ||
| } | ||
| if (tf === 0) continue; | ||
| const numerator = tf * (BM25_K1 + 1); | ||
| const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl); | ||
| score += effectiveIdf * (numerator / denominator); | ||
| } | ||
| return score; | ||
| } | ||
| function searchEngrams(engrams, query, limit = 20, stats) { | ||
| const queryTokens = ftsTokenize(query); | ||
| if (queryTokens.length === 0) return []; | ||
| const idfWeights = computeIdf(engrams, queryTokens, stats); | ||
| const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0; | ||
| let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0); | ||
| if (scored.length === 0) { | ||
| scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0); | ||
| } | ||
| return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram); | ||
| } | ||
| export { | ||
| MIN_TOKEN_LENGTH, | ||
| TOKENIZER_VERSION, | ||
| ftsTokenize, | ||
| engramSearchText, | ||
| embeddingContentHash, | ||
| hashEmbeddedText, | ||
| termMatches, | ||
| computeIdf, | ||
| extendCorpusStats, | ||
| ftsScore, | ||
| searchEngrams | ||
| }; |
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/embedders/transformers-base.ts | ||
| var pipelineCache = /* @__PURE__ */ new Map(); | ||
| async function loadPipeline(modelId, dtype) { | ||
| const key = `${modelId}::${dtype ?? "fp32"}`; | ||
| let pending = pipelineCache.get(key); | ||
| if (!pending) { | ||
| pending = (async () => { | ||
| process.env.HF_HUB_DISABLE_XET ??= "1"; | ||
| const { pipeline } = await import("@huggingface/transformers"); | ||
| return pipeline("feature-extraction", modelId, dtype ? { dtype } : void 0); | ||
| })(); | ||
| pipelineCache.set(key, pending); | ||
| } | ||
| return await pending; | ||
| } | ||
| function makeTransformersAdapter(config) { | ||
| const pooling = config.pooling; | ||
| const normalize = config.normalize ?? true; | ||
| async function embedOne(text) { | ||
| const pipe = await loadPipeline(config.modelId, config.dtype); | ||
| const result = await pipe(text, { pooling, normalize }); | ||
| const arr = result.data instanceof Float32Array ? result.data : new Float32Array(result.data); | ||
| if (arr.length !== config.dim) { | ||
| throw new Error( | ||
| `Embedder "${config.name}" returned ${arr.length}-dim vector, expected ${config.dim}` | ||
| ); | ||
| } | ||
| return arr; | ||
| } | ||
| return { | ||
| name: config.name, | ||
| dim: config.dim, | ||
| modelId: config.modelId, | ||
| embed: embedOne, | ||
| async embedBatch(texts) { | ||
| const out = []; | ||
| for (const t of texts) out.push(await embedOne(t)); | ||
| return out; | ||
| } | ||
| }; | ||
| } | ||
| // src/embedders/minilm.ts | ||
| var MINILM_MODEL_ID = "Xenova/all-MiniLM-L6-v2"; | ||
| function makeMiniLMAdapter() { | ||
| return makeTransformersAdapter({ | ||
| name: "minilm", | ||
| dim: 384, | ||
| modelId: MINILM_MODEL_ID, | ||
| pooling: "mean", | ||
| normalize: true, | ||
| dtype: "fp32" | ||
| }); | ||
| } | ||
| // src/embedders/bge-small.ts | ||
| var BGE_SMALL_MODEL_ID = "Xenova/bge-small-en-v1.5"; | ||
| function makeBgeSmallAdapter() { | ||
| return makeTransformersAdapter({ | ||
| name: "bge-small", | ||
| dim: 384, | ||
| modelId: BGE_SMALL_MODEL_ID, | ||
| pooling: "cls", | ||
| normalize: true, | ||
| dtype: "fp32" | ||
| }); | ||
| } | ||
| // src/embedders/bge-base.ts | ||
| var BGE_BASE_MODEL_ID = "Xenova/bge-base-en-v1.5"; | ||
| function makeBgeBaseAdapter() { | ||
| return makeTransformersAdapter({ | ||
| name: "bge-base", | ||
| dim: 768, | ||
| modelId: BGE_BASE_MODEL_ID, | ||
| pooling: "cls", | ||
| normalize: true, | ||
| dtype: "fp32" | ||
| }); | ||
| } | ||
| // src/embedders/embedding-gemma.ts | ||
| var EMBEDDING_GEMMA_MODEL_ID = "onnx-community/embeddinggemma-300m-ONNX"; | ||
| var DIM = 768; | ||
| var loaded = null; | ||
| async function load() { | ||
| if (!loaded) { | ||
| loaded = (async () => { | ||
| process.env.HF_HUB_DISABLE_XET ??= "1"; | ||
| const { AutoTokenizer, AutoModel } = await import("@huggingface/transformers"); | ||
| const tokenizer = await AutoTokenizer.from_pretrained(EMBEDDING_GEMMA_MODEL_ID); | ||
| const model = await AutoModel.from_pretrained(EMBEDDING_GEMMA_MODEL_ID, { dtype: "q8" }); | ||
| return { tokenizer, model }; | ||
| })(); | ||
| } | ||
| return loaded; | ||
| } | ||
| function makeEmbeddingGemmaAdapter() { | ||
| async function embedOne(text, role) { | ||
| const prefix = role === "query" ? "task: search result | query: " : "title: none | text: "; | ||
| const { tokenizer, model } = await load(); | ||
| const inputs = await tokenizer(prefix + text, { padding: true, truncation: true }); | ||
| const outputs = await model(inputs); | ||
| const raw = outputs.sentence_embedding.data; | ||
| const arr = raw instanceof Float32Array ? raw : new Float32Array(raw); | ||
| if (arr.length !== DIM) { | ||
| throw new Error(`EmbeddingGemma: expected ${DIM}-dim sentence_embedding, got ${arr.length}`); | ||
| } | ||
| return arr; | ||
| } | ||
| return { | ||
| // Suffix is a cache-space marker: embeddings.ts detects a name change and | ||
| // auto-invalidates any cached vectors built in a different space. '@graph' | ||
| // marked the switch off JS-side pooling; '-mcprefix' marks the #483 switch to | ||
| // the model-card role prefixes (old "query:"/"passage:" vectors are a | ||
| // different space and must be rebuilt). | ||
| name: "embedding-gemma@graph-mcprefix", | ||
| dim: DIM, | ||
| modelId: EMBEDDING_GEMMA_MODEL_ID, | ||
| embed: embedOne, | ||
| async embedBatch(texts) { | ||
| const out = []; | ||
| for (const t of texts) out.push(await embedOne(t)); | ||
| return out; | ||
| } | ||
| }; | ||
| } | ||
| // src/embedders/openai.ts | ||
| var OPENAI_3_LARGE_MODEL_ID = "text-embedding-3-large"; | ||
| var OPENAI_3_LARGE_DIM = 3072; | ||
| var OPENAI_3_LARGE_MAX_INPUT_TOKENS = 8191; | ||
| var ENDPOINT = "https://api.openai.com/v1/embeddings"; | ||
| var DEFAULT_MAX_BATCH_SIZE = 2048; | ||
| var DEFAULT_MAX_TOKENS_PER_REQUEST = 3e5; | ||
| var DEFAULT_TIMEOUT_MS = 3e4; | ||
| var DEFAULT_MAX_RETRIES = 3; | ||
| var DEFAULT_RETRY_BASE_MS = 1e3; | ||
| var MAX_RETRY_AFTER_MS = 6e4; | ||
| var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 503]); | ||
| function estimateTokens(text) { | ||
| return Math.ceil(text.length / 4); | ||
| } | ||
| function parseRetryAfterMs(header) { | ||
| if (!header) return null; | ||
| const secs = Number(header); | ||
| if (Number.isFinite(secs) && secs >= 0) { | ||
| return Math.min(secs * 1e3, MAX_RETRY_AFTER_MS); | ||
| } | ||
| const dateMs = Date.parse(header); | ||
| if (!Number.isNaN(dateMs)) { | ||
| return Math.min(Math.max(dateMs - Date.now(), 0), MAX_RETRY_AFTER_MS); | ||
| } | ||
| return null; | ||
| } | ||
| function readApiKey() { | ||
| const key = process.env.OPENAI_API_KEY; | ||
| if (!key || key.trim() === "") { | ||
| throw new Error( | ||
| "openai-3-large embedder requires OPENAI_API_KEY. Set the env var (export OPENAI_API_KEY=sk-...) or unset PLUR_EMBEDDER to fall back to the local default." | ||
| ); | ||
| } | ||
| return key; | ||
| } | ||
| function clampToTokenLimit(text, index) { | ||
| const maxChars = OPENAI_3_LARGE_MAX_INPUT_TOKENS * 4; | ||
| if (text.length <= maxChars) return text; | ||
| logger.warning( | ||
| `[openai-3-large] input ${index} is ~${estimateTokens(text)} tokens (limit ${OPENAI_3_LARGE_MAX_INPUT_TOKENS}) \u2014 truncating to avoid an HTTP 400 mid-batch` | ||
| ); | ||
| return text.slice(0, maxChars); | ||
| } | ||
| function parseVectors(json, expected) { | ||
| const data = json.data; | ||
| if (!Array.isArray(data) || data.length !== expected) { | ||
| throw new Error( | ||
| `openai-3-large embed returned ${data?.length ?? "no"} vectors for ${expected} inputs` | ||
| ); | ||
| } | ||
| return data.map((row, i) => { | ||
| const e = row.embedding; | ||
| if (!Array.isArray(e)) { | ||
| throw new Error(`openai-3-large returned non-array embedding at index ${i}`); | ||
| } | ||
| if (e.length !== OPENAI_3_LARGE_DIM) { | ||
| throw new Error( | ||
| `openai-3-large returned ${e.length}-dim vector at index ${i}, expected ${OPENAI_3_LARGE_DIM}` | ||
| ); | ||
| } | ||
| return Float32Array.from(e); | ||
| }); | ||
| } | ||
| async function postEmbed(texts, opts) { | ||
| const key = readApiKey(); | ||
| for (let attempt = 0; ; attempt++) { | ||
| const controller = new AbortController(); | ||
| const timer = setTimeout(() => controller.abort(), opts.timeoutMs); | ||
| let res; | ||
| try { | ||
| res = await opts.fetch(ENDPOINT, { | ||
| method: "POST", | ||
| headers: { | ||
| "content-type": "application/json", | ||
| authorization: `Bearer ${key}` | ||
| }, | ||
| body: JSON.stringify({ | ||
| model: OPENAI_3_LARGE_MODEL_ID, | ||
| input: texts | ||
| }), | ||
| signal: controller.signal | ||
| }); | ||
| } catch (err) { | ||
| if (controller.signal.aborted) { | ||
| throw new Error(`openai-3-large embed timed out after ${opts.timeoutMs}ms`); | ||
| } | ||
| throw err; | ||
| } finally { | ||
| clearTimeout(timer); | ||
| } | ||
| if (res.ok) { | ||
| return parseVectors(await res.json(), texts.length); | ||
| } | ||
| const body = await res.text().catch(() => ""); | ||
| if (RETRYABLE_STATUS.has(res.status) && attempt < opts.maxRetries) { | ||
| const delay = parseRetryAfterMs(res.headers.get("retry-after")) ?? opts.retryBaseMs * 2 ** attempt; | ||
| logger.warning( | ||
| `[openai-3-large] HTTP ${res.status} \u2014 retrying in ${delay}ms (attempt ${attempt + 1}/${opts.maxRetries})` | ||
| ); | ||
| await new Promise((r) => setTimeout(r, delay)); | ||
| continue; | ||
| } | ||
| throw new Error( | ||
| `openai-3-large embed failed: HTTP ${res.status} ${res.statusText}${body ? ` \u2014 ${body.slice(0, 200)}` : ""}` | ||
| ); | ||
| } | ||
| } | ||
| function chunkInputs(texts, opts) { | ||
| const chunks = []; | ||
| let current = []; | ||
| let currentTokens = 0; | ||
| for (const text of texts) { | ||
| const cost = estimateTokens(text); | ||
| if (current.length > 0 && (current.length >= opts.maxBatchSize || currentTokens + cost > opts.maxTokensPerRequest)) { | ||
| chunks.push(current); | ||
| current = []; | ||
| currentTokens = 0; | ||
| } | ||
| current.push(text); | ||
| currentTokens += cost; | ||
| } | ||
| if (current.length > 0) chunks.push(current); | ||
| return chunks; | ||
| } | ||
| function makeOpenAI3LargeAdapter(options) { | ||
| const opts = { | ||
| fetch: options?.fetch ?? ((...args) => globalThis.fetch(...args)), | ||
| timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, | ||
| maxRetries: options?.maxRetries ?? DEFAULT_MAX_RETRIES, | ||
| retryBaseMs: options?.retryBaseMs ?? DEFAULT_RETRY_BASE_MS, | ||
| maxBatchSize: options?.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE, | ||
| maxTokensPerRequest: options?.maxTokensPerRequest ?? DEFAULT_MAX_TOKENS_PER_REQUEST | ||
| }; | ||
| return { | ||
| name: "openai-3-large", | ||
| dim: OPENAI_3_LARGE_DIM, | ||
| modelId: OPENAI_3_LARGE_MODEL_ID, | ||
| async embed(text) { | ||
| const [v] = await postEmbed([clampToTokenLimit(text, 0)], opts); | ||
| return v; | ||
| }, | ||
| async embedBatch(texts) { | ||
| if (texts.length === 0) return []; | ||
| const clamped = texts.map((t, i) => clampToTokenLimit(t, i)); | ||
| const out = []; | ||
| for (const chunk of chunkInputs(clamped, opts)) { | ||
| out.push(...await postEmbed(chunk, opts)); | ||
| } | ||
| return out; | ||
| } | ||
| }; | ||
| } | ||
| // src/embedders/index.ts | ||
| var EMBEDDER_NAMES = [ | ||
| "minilm", | ||
| "bge-small", | ||
| "bge-base", | ||
| "embedding-gemma", | ||
| "openai-3-large" | ||
| ]; | ||
| var DEFAULT_EMBEDDER = "bge-small"; | ||
| var adapterCache = /* @__PURE__ */ new Map(); | ||
| function _resetEmbedderCache() { | ||
| adapterCache.clear(); | ||
| } | ||
| function getEmbedder(name) { | ||
| if (!EMBEDDER_NAMES.includes(name)) { | ||
| throw new Error(`Unknown embedder "${name}". Known: ${EMBEDDER_NAMES.join(", ")}`); | ||
| } | ||
| let adapter = adapterCache.get(name); | ||
| if (!adapter) { | ||
| adapter = build(name); | ||
| adapterCache.set(name, adapter); | ||
| } | ||
| return adapter; | ||
| } | ||
| function build(name) { | ||
| switch (name) { | ||
| case "minilm": | ||
| return makeMiniLMAdapter(); | ||
| case "bge-small": | ||
| return makeBgeSmallAdapter(); | ||
| case "bge-base": | ||
| return makeBgeBaseAdapter(); | ||
| case "embedding-gemma": | ||
| return makeEmbeddingGemmaAdapter(); | ||
| case "openai-3-large": | ||
| return makeOpenAI3LargeAdapter(); | ||
| default: { | ||
| const _exhaustive = name; | ||
| throw new Error(`Unhandled embedder name: ${String(_exhaustive)}`); | ||
| } | ||
| } | ||
| } | ||
| var warnedUnknown = false; | ||
| function resolveEmbedderName(env = process.env) { | ||
| const raw = env.PLUR_EMBEDDER?.trim(); | ||
| if (!raw) return DEFAULT_EMBEDDER; | ||
| if (EMBEDDER_NAMES.includes(raw)) return raw; | ||
| if (!warnedUnknown) { | ||
| logger.warning( | ||
| `[embedders] PLUR_EMBEDDER="${raw}" not recognised. Falling back to "${DEFAULT_EMBEDDER}". Known: ${EMBEDDER_NAMES.join(", ")}` | ||
| ); | ||
| warnedUnknown = true; | ||
| } | ||
| return DEFAULT_EMBEDDER; | ||
| } | ||
| function _resetResolveWarnings() { | ||
| warnedUnknown = false; | ||
| } | ||
| export { | ||
| EMBEDDER_NAMES, | ||
| DEFAULT_EMBEDDER, | ||
| _resetEmbedderCache, | ||
| getEmbedder, | ||
| resolveEmbedderName, | ||
| _resetResolveWarnings | ||
| }; |
| import { | ||
| atomicWrite, | ||
| withLock | ||
| } from "./chunk-TXHLQGN3.js"; | ||
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/schemas/engram.ts | ||
| import { z } from "zod"; | ||
| var ActivationSchema = z.object({ | ||
| retrieval_strength: z.number().min(0).max(1), | ||
| storage_strength: z.number().min(0).max(1), | ||
| frequency: z.number().int().min(0), | ||
| last_accessed: z.string().describe("Date or ISO 8601 timestamp of last access.") | ||
| }).describe("ACT-R activation parameters driving decay and ranking. STABLE."); | ||
| var KnowledgeTypeSchema = z.object({ | ||
| memory_class: z.enum(["semantic", "episodic", "procedural", "metacognitive"]), | ||
| cognitive_level: z.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"]).describe("Bloom's taxonomy level.") | ||
| }); | ||
| var KnowledgeAnchorSchema = z.object({ | ||
| path: z.string().describe("Path to a grounding document/file."), | ||
| relevance: z.enum(["primary", "supporting", "example"]).default("supporting"), | ||
| snippet: z.string().max(200).optional(), | ||
| snippet_extracted_at: z.string().optional() | ||
| }); | ||
| var AssociationSchema = z.object({ | ||
| target_type: z.enum(["engram", "document"]), | ||
| target: z.string().describe("ID or path of the association target."), | ||
| strength: z.number().min(0).max(0.95), | ||
| type: z.enum(["semantic", "temporal", "causal", "co_accessed"]), | ||
| updated_at: z.string().optional() | ||
| }); | ||
| var DualCodingSchema = z.object({ | ||
| example: z.string().optional(), | ||
| analogy: z.string().optional() | ||
| }).describe("Worked example and/or analogy (dual coding). At least one of example or analogy MUST be provided (enforced at runtime by the Zod .refine below).").refine( | ||
| (d) => d.example || d.analogy, | ||
| "At least one of example or analogy must be provided" | ||
| ); | ||
| var RelationsSchema = z.object({ | ||
| broader: z.array(z.string()).default([]), | ||
| narrower: z.array(z.string()).default([]), | ||
| related: z.array(z.string()).default([]), | ||
| conflicts: z.array(z.string()).default([]), | ||
| /** IDs of engrams this one intentionally replaces (#240). An intentional | ||
| * update is not a tension — the scanner skips supersedes-linked pairs. */ | ||
| supersedes: z.array(z.string()).default([]), | ||
| /** Reverse edge of `supersedes` (#240) — IDs of engrams that replace this one. */ | ||
| superseded_by: z.array(z.string()).default([]) | ||
| }).describe("Typed graph edges between engram IDs."); | ||
| var ProvenanceSchema = z.object({ | ||
| origin: z.string(), | ||
| chain: z.array(z.string()).default([]), | ||
| signature: z.string().nullable().default(null).describe("RESERVED. Detached signature over the engram. Algorithm and canonicalization not yet specified \u2014 see ENGRAM-STANDARD-v1.md \xA77."), | ||
| license: z.string().default("cc-by-sa-4.0") | ||
| }).describe("Origin and signing chain. STABLE for origin/chain/license; signature is RESERVED (see ENGRAM-STANDARD-v1.md \xA77)."); | ||
| var FeedbackSignalsSchema = z.object({ | ||
| positive: z.number().int().default(0), | ||
| negative: z.number().int().default(0), | ||
| neutral: z.number().int().default(0) | ||
| }); | ||
| var EntityRefSchema = z.object({ | ||
| name: z.string(), | ||
| type: z.enum([ | ||
| "person", | ||
| "organization", | ||
| "technology", | ||
| "concept", | ||
| "project", | ||
| "tool", | ||
| "place", | ||
| "event", | ||
| "standard", | ||
| "other" | ||
| ]), | ||
| uri: z.string().url().optional() | ||
| }); | ||
| var TemporalSchema = z.object({ | ||
| learned_at: z.string(), | ||
| valid_from: z.string().optional(), | ||
| valid_until: z.string().optional(), | ||
| ingested_at: z.string().optional() | ||
| }).describe("Bi-temporal anchoring (Zep-inspired). When is this knowledge true?"); | ||
| var UsageStatsSchema = z.object({ | ||
| injections: z.number().int().default(0), | ||
| hits: z.number().int().default(0), | ||
| misses: z.number().int().default(0), | ||
| last_hit_at: z.string().optional() | ||
| }); | ||
| var EpisodicFieldsSchema = z.object({ | ||
| emotional_weight: z.number().int().min(1).max(10).default(5), | ||
| confidence: z.number().int().min(1).max(10).default(5), | ||
| trigger_context: z.string().optional(), | ||
| journal_ref: z.string().optional() | ||
| }); | ||
| var PreviousVersionRefSchema = z.object({ | ||
| event_id: z.string(), | ||
| changed_at: z.string() | ||
| }); | ||
| var ExchangeMetadataSchema = z.object({ | ||
| fitness_score: z.number().min(0).max(1).optional(), | ||
| environmental_diversity: z.number().int().default(0), | ||
| adoption_count: z.number().int().default(0), | ||
| contradiction_rate: z.number().min(0).max(1).default(0) | ||
| }); | ||
| var SerendipitySchema = z.object({ | ||
| unexpectedness: z.number().min(0).max(1), | ||
| relevance: z.number().min(0).max(1), | ||
| score: z.number().min(0).max(1) | ||
| }); | ||
| var InsightFateSchema = z.enum([ | ||
| "surfaced", | ||
| // shown in a briefing; no downstream action yet | ||
| "promoted", | ||
| // became a durable engram / zettel | ||
| "cited", | ||
| // referenced in later journal/work | ||
| "tasked", | ||
| // converted to a GTD task | ||
| "dismissed", | ||
| // user/LLM rejected it | ||
| "expired" | ||
| // decayed out of the buffer unused | ||
| ]); | ||
| var InsightFieldSchema = z.object({ | ||
| /** Which memory-stream operation produced this insight. Nightly arc: | ||
| * `distill` (episode→insight synthesis) → `consolidate` (convergent gist | ||
| * abstraction over the buffer) → `dream` (divergent REM-style recombination — | ||
| * speculative, never auto-promoted). `connect`/`emerge`/`drift` are on-demand lenses. */ | ||
| operation: z.enum(["distill", "consolidate", "dream", "connect", "emerge", "drift"]), | ||
| synthesized_at: z.string(), | ||
| /** Anti-hallucination grounding. Cited source notes live in the parent engram's | ||
| * `knowledge_anchors[]`; this flags whether the claim was verified against those | ||
| * snippets. `ungrounded` = couldn't cite sources → quarantined (`candidate`, never | ||
| * surfaced). `speculative` = a `dream`: its recombined INPUTS are cited but its | ||
| * CONCLUSION is an explicit hypothesis — surfaced only as inspiration, and (per the | ||
| * promote-requires-grounding refine below) it must be re-grounded to `verified` | ||
| * before it can be promoted to a durable engram. */ | ||
| grounding: z.enum(["verified", "unverified", "ungrounded", "speculative"]).default("unverified"), | ||
| /** The episode-log slice this insight was distilled from (evidence trail). */ | ||
| source_episode_ids: z.array(z.string()).default([]), | ||
| /** Distinct objective for connect/emerge/dream insights. */ | ||
| serendipity: SerendipitySchema.optional(), | ||
| fate: InsightFateSchema.default("surfaced"), | ||
| /** Engram id / zettel path / task id the insight became, if acted upon. */ | ||
| fate_ref: z.string().optional(), | ||
| fate_at: z.string().optional(), | ||
| /** How many briefings have surfaced this insight (acted-upon-rate denominator). */ | ||
| surfaced_count: z.number().int().min(0).default(0) | ||
| }).refine( | ||
| // Promote-requires-grounding (user rule 2026-06-15): a dream is inspiration, not | ||
| // fact. A speculative/ungrounded insight can only become a durable promotion once | ||
| // it has been re-grounded in reality (grounding=verified). | ||
| (i) => i.fate !== "promoted" || i.grounding === "verified", | ||
| { message: "A promoted insight must be grounded (grounding=verified); speculative dreams cannot be promoted until re-grounded.", path: ["grounding"] } | ||
| ); | ||
| var ExtractionProvenanceSchema = z.object({ | ||
| confidence: z.number().min(0).max(1).optional().describe("0-1 classifier confidence at extraction time. Frozen at write; distinct from feedback-derived computeConfidence() and from episodic.confidence."), | ||
| source_commit: z.string().optional().describe("Git SHA of the source repository at extraction time (reproducibility)."), | ||
| extractor_version: z.string().optional().describe("Version of the extracting CLI/tool (schema-migration handle). Complementary to the pack-level capsule producer field (#61).") | ||
| }).passthrough().describe("ETL extraction provenance convention carried in structured_data.extraction (#463). Not wired into EngramSchema."); | ||
| function getExtractionProvenance(engram) { | ||
| const extraction = engram.structured_data?.["extraction"]; | ||
| if (extraction === void 0 || extraction === null) return null; | ||
| const parsed = ExtractionProvenanceSchema.safeParse(extraction); | ||
| return parsed.success ? parsed.data : null; | ||
| } | ||
| var EngramSchema = z.object({ | ||
| // Identity | ||
| id: z.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/).describe("Unique identifier. Class prefix ENG (concrete engram), ABS (abstraction), or META (meta-engram). Canonical concrete form: ENG-YYYY-MMDD-NNN; store-namespaced form: ENG-{PREFIX}-YYYY-MMDD-NNN."), | ||
| version: z.number().int().min(1).default(2).describe("Schema-shape generation of this engram object (currently 2). Distinct from engram_version, which tracks content evolution."), | ||
| // 'active' and 'retired' are the two states any current code path assigns | ||
| // (retire via forget/dedup/supersede). 'dormant' and 'candidate' are NOT | ||
| // assigned by any code today: 'dormant' was only ever set by the batchDecay | ||
| // pass removed in #563 (decay is now a read-time property, not a materialized | ||
| // status), and 'candidate' is reserved. They are kept in the enum so stores | ||
| // written before #563 that persisted status:'dormant' still load, and so the | ||
| // status filter accepts them; do not remove without a data migration. | ||
| status: z.enum(["active", "dormant", "retired", "candidate"]).describe("Lifecycle state. Assigned values today are active/retired; dormant/candidate are legacy/reserved (see note above)."), | ||
| consolidated: z.boolean().default(false).describe("Whether this engram has been through consolidation (sleep-like batch reprocessing)."), | ||
| type: z.enum(["behavioral", "terminological", "procedural", "architectural"]).describe("Top-level classification of the knowledge."), | ||
| scope: z.string().describe("Hierarchical namespace, e.g. 'global', 'project:my-app', 'group:plur/test'. Free-form string; ':' separates scope kind from path."), | ||
| visibility: z.enum(["private", "public", "template"]).default("private").describe("Sharing posture. 'private' engrams MUST NOT be exported in packs."), | ||
| // Content | ||
| statement: z.string().min(1).describe("The assertion itself \u2014 the load-bearing content of the engram."), | ||
| rationale: z.string().optional().describe("Why this is true / why it matters."), | ||
| contraindications: z.array(z.string()).optional().describe("Conditions under which the statement does NOT apply."), | ||
| // Lineage | ||
| source: z.string().optional().describe("Free-text origin (session, document, conversation)."), | ||
| source_patterns: z.array(z.string()).optional().describe("Pattern IDs that contributed to this engram."), | ||
| derivation_count: z.number().int().min(0).default(1).describe("How many derivation steps produced this engram."), | ||
| pack: z.string().nullable().default(null).describe("Name of the pack this engram belongs to, or null."), | ||
| abstract: z.string().nullable().default(null).describe("ID of an ABS- abstraction this engram instantiates, or null."), | ||
| derived_from: z.string().nullable().default(null).describe("ID of the engram this was derived from, or null."), | ||
| // Classification | ||
| knowledge_type: KnowledgeTypeSchema.optional(), | ||
| domain: z.string().optional().describe("Dotted domain path, e.g. 'dev/testing' or 'plur.session'."), | ||
| tags: z.array(z.string()).default([]).describe("Free-form tags used for matching and retrieval."), | ||
| // Activation (ACT-R model) | ||
| activation: ActivationSchema.default({ | ||
| retrieval_strength: 0.7, | ||
| storage_strength: 1, | ||
| frequency: 0, | ||
| last_accessed: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10) | ||
| }), | ||
| // Relations & grounding | ||
| relations: RelationsSchema.optional(), | ||
| associations: z.array(AssociationSchema).default([]), | ||
| knowledge_anchors: z.array(KnowledgeAnchorSchema).default([]), | ||
| dual_coding: DualCodingSchema.optional(), | ||
| // Provenance | ||
| provenance: ProvenanceSchema.optional(), | ||
| // Feedback | ||
| feedback_signals: FeedbackSignalsSchema.default({ positive: 0, negative: 0, neutral: 0 }), | ||
| // === NEW OPTIONAL FIELDS (v2.1) === | ||
| /** Typed entity references extracted from statement. Enables graph queries. */ | ||
| entities: z.array(EntityRefSchema).optional().describe("Typed entity references extracted from statement. Enables graph queries."), | ||
| /** Temporal validity window. When is this knowledge true? */ | ||
| temporal: TemporalSchema.optional(), | ||
| /** Automatic usage tracking. Injections, hits, misses. */ | ||
| usage: UsageStatsSchema.optional(), | ||
| /** Episodic context: emotional weight, confidence, trigger. */ | ||
| episodic: EpisodicFieldsSchema.optional(), | ||
| /** Exchange marketplace metadata: fitness, adoption, diversity. */ | ||
| exchange: ExchangeMetadataSchema.optional(), | ||
| /** Extensible key-value data for domain-specific fields. */ | ||
| structured_data: z.record(z.string(), z.unknown()).optional().describe("Extensible key-value data for domain-specific fields."), | ||
| /** Memory-stream insight provenance (metacognition Phase 1). Orthogonal to | ||
| * `type`. Present iff this engram was synthesized by the metacognition memory | ||
| * stream; the episodic insight buffer is the set of engrams where this is set. */ | ||
| insight: InsightFieldSchema.optional(), | ||
| /** Polarity classification: 'do' for directives, 'dont' for prohibitions, null for unclassified. */ | ||
| polarity: z.enum(["do", "dont"]).nullable().default(null).describe("'do' for directives, 'dont' for prohibitions, null for unclassified."), | ||
| // === SP1: Memory Intelligence fields === | ||
| content_hash: z.string().optional().describe("Hash of normalized statement content, used for dedup."), | ||
| commitment: z.enum(["exploring", "leaning", "decided", "locked"]).optional().describe("Commitment level of the asserted knowledge."), | ||
| locked_at: z.string().optional().describe("Timestamp when commitment reached 'locked'."), | ||
| locked_reason: z.string().optional().describe("Why this engram was locked."), | ||
| // === SP1: Reference counting (issue #107) === | ||
| /** Number of write attempts that resolved to this engram. | ||
| * Incremented on every hash-dedup hit; decremented by forget(). | ||
| * Engram physically retires only when this reaches 0. */ | ||
| reference_count: z.number().int().min(0).default(1).describe("Number of write attempts that resolved to this engram (same-scope re-learns). Engram retires only when this reaches 0."), | ||
| /** Provenance of each write attempt. One entry per write (including the | ||
| * first). Migrated old engrams without this field start with []. */ | ||
| sources: z.array(z.object({ | ||
| scope: z.string(), | ||
| session_id: z.string().nullable().default(null), | ||
| stored_at: z.string().describe("ISO 8601 timestamp of this write.") | ||
| })).default([]).describe("Provenance of each write attempt; one entry per write."), | ||
| // === SP1: Cross-scope recurrence (issue #176) === | ||
| /** Number of times this engram's content was re-learned at a DIFFERENT | ||
| * scope than the original. Triggers auto-broadening + commitment | ||
| * escalation when threshold is crossed. Distinct from reference_count | ||
| * (which counts re-learns in the SAME scope) — recurrence_count is | ||
| * evidence of universal applicability, not just repetition. */ | ||
| recurrence_count: z.number().int().min(0).default(0).describe("Number of times this content was re-learned at a DIFFERENT scope than the original. Evidence of universal applicability."), | ||
| // === SP2: History & Evolution fields === | ||
| engram_version: z.number().int().min(1).default(1).describe("Content-evolution version (incremented when the statement materially changes)."), | ||
| previous_version_ref: PreviousVersionRefSchema.optional(), | ||
| episode_ids: z.array(z.string()).default([]).describe("IDs of episodes (raw conversational events) that produced or reinforced this engram."), | ||
| // === SP3: Retrieval & Injection fields === | ||
| summary: z.string().max(80).optional().describe("Short (<=80 char) injection-friendly summary."), | ||
| /** | ||
| * Always-load flag. Pinned engrams bypass the term-hits gate in scoreEngram | ||
| * and are eligible for injection on every session start, regardless of | ||
| * keyword overlap with the user's task. Use sparingly: meta-rules, | ||
| * cross-cutting safety conventions, and core operating principles only. | ||
| * Pinned engrams still respect the token budget — they bypass per-pack and | ||
| * per-domain fairness caps in fillTokenBudget so always-load behavior is | ||
| * honored even if a single pack contributes many. | ||
| */ | ||
| pinned: z.boolean().optional().describe("Always-load flag. Pinned engrams bypass the keyword-relevance gate and are eligible for injection every session. Use sparingly.") | ||
| }); | ||
| var EngramSchemaPassthrough = EngramSchema.passthrough(); | ||
| // src/backup.ts | ||
| import * as fs from "fs"; | ||
| import * as path from "path"; | ||
| import { createHash } from "crypto"; | ||
| import * as yaml from "js-yaml"; | ||
| var BACKUP_DIR = "backups"; | ||
| var KEEP_DAILY = 7; | ||
| var KEEP_WEEKLY = 4; | ||
| var SHRINK_TOLERANCE = 0.1; | ||
| function statePath(root) { | ||
| return path.join(root, BACKUP_DIR, ".state.json"); | ||
| } | ||
| function readState(root) { | ||
| try { | ||
| return JSON.parse(fs.readFileSync(statePath(root), "utf8")); | ||
| } catch { | ||
| return {}; | ||
| } | ||
| } | ||
| function writeState(root, state) { | ||
| const p = statePath(root); | ||
| fs.mkdirSync(path.dirname(p), { recursive: true }); | ||
| fs.writeFileSync(p, JSON.stringify(state, null, 2) + "\n", "utf8"); | ||
| } | ||
| function sha256(content) { | ||
| return createHash("sha256").update(content).digest("hex"); | ||
| } | ||
| function validateStore(filePath, lastGoodCount) { | ||
| const failures = []; | ||
| const reasons = []; | ||
| let raw; | ||
| try { | ||
| raw = fs.readFileSync(filePath); | ||
| } catch (err) { | ||
| return { ok: false, failures: ["unreadable"], reasons: [`cannot read ${filePath}: ${err}`], count: null }; | ||
| } | ||
| if (raw.length === 0) { | ||
| return { ok: false, failures: ["empty"], reasons: ["file is 0 bytes"], count: null }; | ||
| } | ||
| if (!raw.toString("utf8").endsWith("\n")) { | ||
| failures.push("truncated"); | ||
| reasons.push("file does not end with a newline \u2014 PLUR's writer always emits one, so this looks cut short"); | ||
| } | ||
| let doc; | ||
| try { | ||
| doc = yaml.load(raw.toString("utf8")); | ||
| } catch (err) { | ||
| return { ok: false, failures: ["unparseable"], reasons: [`YAML parse failed: ${err}`], count: null }; | ||
| } | ||
| if (doc == null || typeof doc !== "object" || Array.isArray(doc) || !Array.isArray(doc.engrams)) { | ||
| return { | ||
| ok: false, | ||
| failures: ["not-a-store"], | ||
| reasons: ["parsed, but is not a mapping with an `engrams` list"], | ||
| count: null | ||
| }; | ||
| } | ||
| const entries = doc.engrams; | ||
| const count = entries.length; | ||
| let invalid = 0; | ||
| const ids = /* @__PURE__ */ new Set(); | ||
| let duplicateIds = 0; | ||
| let missingIds = 0; | ||
| for (const entry of entries) { | ||
| if (!EngramSchemaPassthrough.safeParse(entry).success) invalid++; | ||
| const id = entry?.id; | ||
| if (typeof id !== "string" || id.length === 0) missingIds++; | ||
| else if (ids.has(id)) duplicateIds++; | ||
| else ids.add(id); | ||
| } | ||
| if (invalid > 0) { | ||
| failures.push("invalid-entries"); | ||
| reasons.push(`${invalid} entry/entries fail schema validation`); | ||
| } | ||
| if (missingIds > 0) { | ||
| failures.push("missing-ids"); | ||
| reasons.push(`${missingIds} entry/entries have no id`); | ||
| } | ||
| if (duplicateIds > 0) { | ||
| failures.push("duplicate-ids"); | ||
| reasons.push(`${duplicateIds} duplicate id(s)`); | ||
| } | ||
| if (typeof lastGoodCount === "number" && lastGoodCount > 0) { | ||
| const floor = lastGoodCount * (1 - SHRINK_TOLERANCE); | ||
| if (count < floor) { | ||
| failures.push("shrunk"); | ||
| reasons.push( | ||
| `holds ${count} engram(s) but the last good snapshot held ${lastGoodCount} \u2014 a drop this large is how a truncation looks` | ||
| ); | ||
| } | ||
| } | ||
| return { ok: failures.length === 0, failures, reasons, count }; | ||
| } | ||
| function todayStamp(now) { | ||
| return now.toISOString().slice(0, 10); | ||
| } | ||
| function snapshotPath(root, stamp) { | ||
| return path.join(root, BACKUP_DIR, `engrams-${stamp}.yaml`); | ||
| } | ||
| var doneThisProcess = /* @__PURE__ */ new Set(); | ||
| function maybeDailyBackup(root, storePath, now = /* @__PURE__ */ new Date()) { | ||
| const key = `${root}\0${todayStamp(now)}`; | ||
| if (doneThisProcess.has(key)) return { taken: false, skipped: "already-today" }; | ||
| try { | ||
| if (!fs.existsSync(storePath)) { | ||
| doneThisProcess.add(key); | ||
| return { taken: false, skipped: "no-store" }; | ||
| } | ||
| const state = readState(root); | ||
| const stamp = todayStamp(now); | ||
| if (state.last_backup_date === stamp) { | ||
| doneThisProcess.add(key); | ||
| return { taken: false, skipped: "already-today" }; | ||
| } | ||
| const existing = listBackups(root); | ||
| const strongest = existing.reduce( | ||
| (max, b) => typeof b.count === "number" && (max === void 0 || b.count > max) ? b.count : max, | ||
| void 0 | ||
| ); | ||
| const baseline = state.last_good_count ?? strongest; | ||
| const validity = validateStore(storePath, baseline); | ||
| if (!validity.ok) { | ||
| logger.warning( | ||
| `[plur:backup] refusing to snapshot ${storePath} \u2014 ${validity.reasons.join("; ")}. Your last good backup is unchanged. Run 'plur doctor' to inspect.` | ||
| ); | ||
| return { taken: false, skipped: "invalid", validity }; | ||
| } | ||
| const bytes = fs.readFileSync(storePath); | ||
| const dest = snapshotPath(root, stamp); | ||
| const sameDay = existing.find((b) => b.stamp === stamp); | ||
| if (sameDay && typeof sameDay.count === "number" && (validity.count ?? 0) < sameDay.count) { | ||
| logger.warning( | ||
| `[plur:backup] keeping today's existing snapshot (${sameDay.count} engrams) \u2014 the live store holds ${validity.count}, and replacing a stronger snapshot with a weaker one would discard the better copy. Run 'plur restore --list' to inspect.` | ||
| ); | ||
| doneThisProcess.add(key); | ||
| return { taken: false, skipped: "invalid", validity }; | ||
| } | ||
| fs.mkdirSync(path.dirname(dest), { recursive: true }); | ||
| writeFileDurable(dest, bytes); | ||
| writeFileDurable( | ||
| `${dest}.sha256`, | ||
| Buffer.from( | ||
| `${sha256(bytes)} ${path.basename(dest)} | ||
| ${validity.count} engrams | ||
| taken_at ${now.toISOString()} | ||
| `, | ||
| "utf8" | ||
| ) | ||
| ); | ||
| doneThisProcess.add(key); | ||
| writeState(root, { | ||
| last_backup_date: stamp, | ||
| last_good_count: validity.count ?? void 0, | ||
| last_good_sha256: sha256(bytes) | ||
| }); | ||
| rotate(root, now); | ||
| return { taken: true, path: dest, validity }; | ||
| } catch (err) { | ||
| logger.warning(`[plur:backup] snapshot failed (the write itself was unaffected): ${err}`); | ||
| return { taken: false, skipped: "invalid" }; | ||
| } | ||
| } | ||
| function flushFileAt(filePath) { | ||
| let fd; | ||
| try { | ||
| fd = fs.openSync(filePath, "r+"); | ||
| fs.fsyncSync(fd); | ||
| } catch { | ||
| } finally { | ||
| if (fd !== void 0) { | ||
| try { | ||
| fs.closeSync(fd); | ||
| } catch { | ||
| } | ||
| } | ||
| } | ||
| } | ||
| function writeFileDurable(dest, bytes) { | ||
| const fd = fs.openSync(dest, "w"); | ||
| try { | ||
| fs.writeFileSync(fd, bytes); | ||
| fs.fsyncSync(fd); | ||
| } finally { | ||
| fs.closeSync(fd); | ||
| } | ||
| } | ||
| function listBackups(root) { | ||
| const dir = path.join(root, BACKUP_DIR); | ||
| if (!fs.existsSync(dir)) return []; | ||
| const out = []; | ||
| for (const name of fs.readdirSync(dir)) { | ||
| const m = name.match(/^engrams-(\d{4}-\d{2}-\d{2})\.yaml$/); | ||
| if (!m) continue; | ||
| const full = path.join(dir, name); | ||
| const entry = { path: full, stamp: m[1], size: fs.statSync(full).size }; | ||
| try { | ||
| const sidecar = fs.readFileSync(`${full}.sha256`, "utf8"); | ||
| entry.sha256 = sidecar.split(/\s+/)[0]; | ||
| const cm = sidecar.match(/(\d+) engrams/); | ||
| if (cm) entry.count = parseInt(cm[1], 10); | ||
| const tm = sidecar.match(/taken_at (\S+)/); | ||
| if (tm) entry.takenAt = tm[1]; | ||
| } catch { | ||
| } | ||
| out.push(entry); | ||
| } | ||
| return out.sort((a, b) => a.stamp < b.stamp ? 1 : -1); | ||
| } | ||
| function rotate(root, now) { | ||
| const all = listBackups(root); | ||
| if (all.length <= KEEP_DAILY) return; | ||
| const keep = /* @__PURE__ */ new Set(); | ||
| for (const b of all.slice(0, KEEP_DAILY)) keep.add(b.path); | ||
| const weeksSeen = /* @__PURE__ */ new Set(); | ||
| for (const b of all.slice(KEEP_DAILY)) { | ||
| const week = isoWeek(/* @__PURE__ */ new Date(`${b.stamp}T00:00:00Z`)); | ||
| if (weeksSeen.has(week)) continue; | ||
| weeksSeen.add(week); | ||
| if (weeksSeen.size <= KEEP_WEEKLY) keep.add(b.path); | ||
| } | ||
| for (const b of all) { | ||
| if (keep.has(b.path)) continue; | ||
| try { | ||
| fs.unlinkSync(b.path); | ||
| fs.unlinkSync(`${b.path}.sha256`); | ||
| } catch { | ||
| } | ||
| } | ||
| void now; | ||
| } | ||
| function isoWeek(d) { | ||
| const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); | ||
| const day = t.getUTCDay() || 7; | ||
| t.setUTCDate(t.getUTCDate() + 4 - day); | ||
| const yearStart = new Date(Date.UTC(t.getUTCFullYear(), 0, 1)); | ||
| const week = Math.ceil(((t.getTime() - yearStart.getTime()) / 864e5 + 1) / 7); | ||
| return `${t.getUTCFullYear()}-W${week}`; | ||
| } | ||
| function planRestore(root, storePath, stamp) { | ||
| const all = listBackups(root); | ||
| if (all.length === 0) throw new Error(`[plur] no backups found in ${path.join(root, BACKUP_DIR)}`); | ||
| const backup = stamp ? all.find((b) => b.stamp === stamp) : all[0]; | ||
| if (!backup) throw new Error(`[plur] no backup for ${stamp}. Available: ${all.map((b) => b.stamp).join(", ")}`); | ||
| const bytes = fs.readFileSync(backup.path); | ||
| const actualSha256 = sha256(bytes); | ||
| const integrityOk = backup.sha256 === void 0 ? false : backup.sha256 === actualSha256; | ||
| const validity = validateStore(backup.path); | ||
| const backupIds = new Set(idsIn(backup.path)); | ||
| const currentIds = idsIn(storePath); | ||
| const wouldLose = currentIds.filter((id) => !backupIds.has(id)); | ||
| return { | ||
| backup, | ||
| validity, | ||
| actualSha256, | ||
| integrityOk, | ||
| wouldLose, | ||
| // Compare against the snapshot's INSTANT where we have it. Falling back to | ||
| // the end of its day is the conservative direction when a sidecar predates | ||
| // this field: it under-reports rather than inventing losses. | ||
| unrecoverable: idsCreatedAfter(root, backup.takenAt ?? `${backup.stamp}T23:59:59.999Z`).filter((id) => !backupIds.has(id)) | ||
| }; | ||
| } | ||
| function idsIn(filePath) { | ||
| try { | ||
| const doc = yaml.load(fs.readFileSync(filePath, "utf8")); | ||
| if (!doc || !Array.isArray(doc.engrams)) return []; | ||
| return doc.engrams.map((e) => e?.id).filter((id) => typeof id === "string"); | ||
| } catch { | ||
| return []; | ||
| } | ||
| } | ||
| function idsCreatedAfter(root, since) { | ||
| const dir = path.join(root, "history"); | ||
| if (!fs.existsSync(dir)) return []; | ||
| const ids = []; | ||
| for (const name of fs.readdirSync(dir)) { | ||
| if (!name.endsWith(".jsonl")) continue; | ||
| let lines; | ||
| try { | ||
| lines = fs.readFileSync(path.join(dir, name), "utf8").split("\n"); | ||
| } catch { | ||
| continue; | ||
| } | ||
| for (const line of lines) { | ||
| if (!line.trim()) continue; | ||
| try { | ||
| const ev = JSON.parse(line); | ||
| if (typeof ev?.timestamp !== "string" || ev.timestamp <= since) continue; | ||
| if (typeof ev?.engram_id === "string") ids.push(ev.engram_id); | ||
| } catch { | ||
| } | ||
| } | ||
| } | ||
| return [...new Set(ids)]; | ||
| } | ||
| function restoreBackup(root, storePath, opts = {}) { | ||
| let plan; | ||
| const superseded = `${storePath}.superseded-${Date.now()}`; | ||
| withLock(storePath, () => { | ||
| plan = planRestore(root, storePath, opts.stamp); | ||
| if (!opts.force) { | ||
| const problems = []; | ||
| if (!plan.validity.ok) problems.push(...plan.validity.reasons); | ||
| if (!plan.integrityOk) { | ||
| problems.push( | ||
| plan.backup.sha256 === void 0 ? "no sha256 sidecar \u2014 cannot verify the backup is intact" : "sha256 does not match the sidecar \u2014 the backup itself is damaged" | ||
| ); | ||
| } | ||
| if (problems.length > 0) { | ||
| throw new Error( | ||
| `[plur] refusing to restore ${plan.backup.path}: ${problems.join("; ")}. | ||
| Restoring is a whole-corpus overwrite; doing it from a backup that does not verify would replace a damaged store with a differently damaged one. | ||
| Pass force to override if you have inspected the file yourself.` | ||
| ); | ||
| } | ||
| } | ||
| if (fs.existsSync(storePath)) { | ||
| fs.copyFileSync(storePath, superseded); | ||
| flushFileAt(superseded); | ||
| } | ||
| atomicWrite(storePath, fs.readFileSync(plan.backup.path, "utf8")); | ||
| }); | ||
| if (plan.wouldLose.length > 0) { | ||
| logger.warning( | ||
| `[plur:restore] ${plan.wouldLose.length} engram(s) present before the restore are not in this backup: ${plan.wouldLose.slice(0, 10).join(", ")}${plan.wouldLose.length > 10 ? ", \u2026" : ""}. The pre-restore store was kept at ${superseded}.` | ||
| ); | ||
| } | ||
| if (plan.unrecoverable.length > 0) { | ||
| logger.warning( | ||
| `[plur:restore] history records ${plan.unrecoverable.length} engram(s) created after this backup that it does not contain: ${plan.unrecoverable.slice(0, 10).join(", ")}${plan.unrecoverable.length > 10 ? ", \u2026" : ""}.` | ||
| ); | ||
| } | ||
| return { ...plan, restored: true, supersededPath: superseded }; | ||
| } | ||
| // src/history.ts | ||
| import * as fs2 from "fs"; | ||
| import { join as join2 } from "path"; | ||
| import { createHash as createHash2 } from "crypto"; | ||
| function appendHistory(root, event) { | ||
| const historyDir = join2(root, "history"); | ||
| if (!fs2.existsSync(historyDir)) { | ||
| fs2.mkdirSync(historyDir, { recursive: true }); | ||
| } | ||
| const date = event.timestamp.slice(0, 7); | ||
| const filePath = join2(historyDir, `${date}.jsonl`); | ||
| const line = JSON.stringify(event) + "\n"; | ||
| const fd = fs2.openSync(filePath, "a"); | ||
| try { | ||
| fs2.writeSync(fd, line); | ||
| try { | ||
| fs2.fsyncSync(fd); | ||
| } catch { | ||
| } | ||
| } finally { | ||
| fs2.closeSync(fd); | ||
| } | ||
| } | ||
| function readHistory(root, yearMonth) { | ||
| const filePath = join2(root, "history", `${yearMonth}.jsonl`); | ||
| if (!fs2.existsSync(filePath)) return []; | ||
| const content = fs2.readFileSync(filePath, "utf8"); | ||
| const lines = content.split("\n").filter((l) => l.trim().length > 0); | ||
| const events = []; | ||
| for (const line of lines) { | ||
| try { | ||
| events.push(JSON.parse(line)); | ||
| } catch { | ||
| } | ||
| } | ||
| return events; | ||
| } | ||
| function listHistoryMonths(root) { | ||
| const historyDir = join2(root, "history"); | ||
| if (!fs2.existsSync(historyDir)) return []; | ||
| return fs2.readdirSync(historyDir).filter((f) => f.endsWith(".jsonl")).map((f) => f.replace(".jsonl", "")).sort(); | ||
| } | ||
| function readHistoryForEngram(root, engramId) { | ||
| const months = listHistoryMonths(root); | ||
| const events = []; | ||
| for (const month of months) { | ||
| const monthEvents = readHistory(root, month); | ||
| for (const event of monthEvents) { | ||
| if (event.engram_id === engramId) { | ||
| events.push(event); | ||
| } | ||
| } | ||
| } | ||
| return events; | ||
| } | ||
| var _PROC_SALT = (process.pid % 1296).toString(36).padStart(2, "0"); | ||
| var _evtSeq = 0; | ||
| var _injSeq = 0; | ||
| function generateEventId() { | ||
| return `EVT-${Date.now()}-${_PROC_SALT}${(_evtSeq++).toString(36).padStart(4, "0")}`; | ||
| } | ||
| function generateInjectionId() { | ||
| return `INJ-${Date.now()}-${_PROC_SALT}${(_injSeq++).toString(36).padStart(4, "0")}`; | ||
| } | ||
| function computeQueryHash(task) { | ||
| const normalized = task.toLowerCase().replace(/\s+/g, " ").trim(); | ||
| return createHash2("sha256").update(normalized).digest("hex").slice(0, 16); | ||
| } | ||
| function findLatestInjectionFor(root, engramId, maxMonths = 2) { | ||
| const now = /* @__PURE__ */ new Date(); | ||
| const allowed = /* @__PURE__ */ new Set(); | ||
| for (let i = 0; i < maxMonths; i++) { | ||
| const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1)); | ||
| allowed.add(d.toISOString().slice(0, 7)); | ||
| } | ||
| const months = listHistoryMonths(root).filter((m) => allowed.has(m)).reverse(); | ||
| for (const month of months) { | ||
| let latest = null; | ||
| for (const event of readHistory(root, month)) { | ||
| if (event.event !== "co_injection") continue; | ||
| const ids = event.data.ids; | ||
| if (!Array.isArray(ids) || !ids.includes(engramId)) continue; | ||
| if (!latest || event.timestamp > latest.timestamp) latest = event; | ||
| } | ||
| if (latest) return { injection_id: latest.engram_id, timestamp: latest.timestamp }; | ||
| } | ||
| return null; | ||
| } | ||
| var INJECTION_SOURCES = /* @__PURE__ */ new Set([ | ||
| "session_start", | ||
| "inject", | ||
| "hook", | ||
| "unknown" | ||
| ]); | ||
| var ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/; | ||
| function readCoInjections(root, months) { | ||
| const events = []; | ||
| let skipped = 0; | ||
| const wanted = months ? new Set(months) : null; | ||
| for (const month of listHistoryMonths(root)) { | ||
| if (wanted && !wanted.has(month)) continue; | ||
| for (const event of readHistory(root, month)) { | ||
| if (event.event !== "co_injection") continue; | ||
| const raw = event.data; | ||
| if (!Array.isArray(raw.ids) || typeof raw.query_hash !== "string") { | ||
| skipped++; | ||
| continue; | ||
| } | ||
| if (typeof event.timestamp !== "string" || !ISO_TIMESTAMP.test(event.timestamp)) { | ||
| skipped++; | ||
| continue; | ||
| } | ||
| const ids = raw.ids.filter((id) => typeof id === "string" && id.length > 0); | ||
| if (ids.length !== raw.ids.length) skipped++; | ||
| const data = { ids, query_hash: raw.query_hash }; | ||
| if (typeof raw.tokens_used === "number" && Number.isFinite(raw.tokens_used)) { | ||
| data.tokens_used = raw.tokens_used; | ||
| } | ||
| if (raw.source !== void 0) { | ||
| data.source = INJECTION_SOURCES.has(raw.source) ? raw.source : "unknown"; | ||
| } | ||
| if (typeof raw.scope === "string") data.scope = raw.scope; | ||
| if (typeof raw.session_id === "string") data.session_id = raw.session_id; | ||
| events.push({ injection_id: event.engram_id, timestamp: event.timestamp, data }); | ||
| } | ||
| } | ||
| events.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); | ||
| return { events, skipped }; | ||
| } | ||
| function countInjectionEvents(root) { | ||
| const counts = { | ||
| co_injection: 0, | ||
| injection_outcome: 0, | ||
| outcome_positive: 0, | ||
| outcome_negative: 0 | ||
| }; | ||
| for (const month of listHistoryMonths(root)) { | ||
| for (const event of readHistory(root, month)) { | ||
| if (event.event === "co_injection") { | ||
| counts.co_injection++; | ||
| } else if (event.event === "injection_outcome") { | ||
| counts.injection_outcome++; | ||
| if (event.data.signal === "positive") counts.outcome_positive++; | ||
| else if (event.data.signal === "negative") counts.outcome_negative++; | ||
| } | ||
| } | ||
| } | ||
| return counts; | ||
| } | ||
| // src/content-hash.ts | ||
| import { createHash as createHash3 } from "crypto"; | ||
| function normalizeStatement(statement) { | ||
| return statement.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim(); | ||
| } | ||
| function computeContentHash(statement) { | ||
| const normalized = normalizeStatement(statement); | ||
| return createHash3("sha256").update(normalized).digest("hex"); | ||
| } | ||
| // src/dedup.ts | ||
| function buildDedupPrompt(newStatement, candidates) { | ||
| const candidateList = candidates.map( | ||
| (c, i) => `${i + 1}. [${c.id}] (${c.type}${c.domain ? ", domain: " + c.domain : ""}) | ||
| "${c.statement}"` | ||
| ).join("\n"); | ||
| return `You are a memory deduplication system. Compare a new memory statement against existing ones. | ||
| NEW STATEMENT: | ||
| "${newStatement}" | ||
| EXISTING ENGRAMS: | ||
| ${candidateList} | ||
| For each existing engram, answer: | ||
| 1. RELATIONSHIP: Is the new statement a DUPLICATE (same meaning), EVOLUTION (updated version of same knowledge), COMPLEMENTARY (related but different angle), or UNRELATED? | ||
| 2. RICHNESS: Does the new statement contain more specific, actionable information than the existing one? (yes/no) | ||
| Then give your OVERALL DECISION (exactly one): | ||
| - NOOP: New statement is an exact duplicate of an existing engram. Return the ID. | ||
| - UPDATE: New statement is an evolution with MORE information. Return the ID to update. | ||
| - MERGE: New statement and an existing one are complementary \u2014 combining them preserves both. Return the ID to merge with. | ||
| - ADD: New statement is genuinely new knowledge. | ||
| Respond in this exact format: | ||
| DECISION: <ADD|UPDATE|MERGE|NOOP> | ||
| TARGET: <engram ID if UPDATE/MERGE/NOOP, or "none" if ADD> | ||
| REASON: <one sentence explanation>`; | ||
| } | ||
| function buildBatchDedupPrompt(statements, existingEngrams) { | ||
| const stmtList = statements.map((s, i) => `${i + 1}. "${s}"`).join("\n"); | ||
| const engramList = existingEngrams.map( | ||
| (e, i) => `${i + 1}. [${e.id}] (${e.type}${e.domain ? ", domain: " + e.domain : ""}) | ||
| "${e.statement}"` | ||
| ).join("\n"); | ||
| return `You are a memory deduplication system. Compare NEW statements against existing engrams. | ||
| NEW STATEMENTS: | ||
| ${stmtList} | ||
| EXISTING ENGRAMS: | ||
| ${engramList} | ||
| For each NEW statement, decide: | ||
| - NOOP: Exact duplicate of an existing engram. | ||
| - UPDATE: Evolution with more info than existing. | ||
| - MERGE: Complementary with existing \u2014 combine. | ||
| - ADD: Genuinely new knowledge. | ||
| Respond with one block per new statement: | ||
| STATEMENT_1: | ||
| DECISION: <ADD|UPDATE|MERGE|NOOP> | ||
| TARGET: <engram ID or "none"> | ||
| STATEMENT_2: | ||
| ...`; | ||
| } | ||
| function parseDedupResponse(response) { | ||
| const decisionMatch = response.match(/DECISION:\s*(ADD|UPDATE|MERGE|NOOP)/i); | ||
| const targetMatch = response.match(/TARGET:\s*([^\n]+)/i); | ||
| const reasonMatch = response.match(/REASON:\s*([^\n]+)/i); | ||
| const decision = decisionMatch?.[1]?.toUpperCase() ?? "ADD"; | ||
| const targetRaw = targetMatch?.[1]?.trim() ?? "none"; | ||
| const target_id = targetRaw === "none" ? null : targetRaw.replace(/[^A-Za-z0-9-]/g, ""); | ||
| const reason = reasonMatch?.[1]?.trim() ?? ""; | ||
| return { decision, target_id, reason }; | ||
| } | ||
| export { | ||
| ExtractionProvenanceSchema, | ||
| getExtractionProvenance, | ||
| EngramSchemaPassthrough, | ||
| BACKUP_DIR, | ||
| validateStore, | ||
| maybeDailyBackup, | ||
| listBackups, | ||
| planRestore, | ||
| restoreBackup, | ||
| appendHistory, | ||
| readHistory, | ||
| listHistoryMonths, | ||
| readHistoryForEngram, | ||
| generateEventId, | ||
| generateInjectionId, | ||
| computeQueryHash, | ||
| findLatestInjectionFor, | ||
| readCoInjections, | ||
| countInjectionEvents, | ||
| normalizeStatement, | ||
| computeContentHash, | ||
| buildDedupPrompt, | ||
| buildBatchDedupPrompt, | ||
| parseDedupResponse | ||
| }; |
| import { | ||
| engramSearchText | ||
| } from "./chunk-SYCJM6JJ.js"; | ||
| import { | ||
| atomicWrite | ||
| } from "./chunk-TXHLQGN3.js"; | ||
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/embeddings.ts | ||
| import { existsSync, readFileSync, mkdirSync } from "fs"; | ||
| import { join, dirname } from "path"; | ||
| import { createHash } from "crypto"; | ||
| var EMBED_DIM = 384; | ||
| var embedPipeline = null; | ||
| var lastLoadError = null; | ||
| var transformersUnavailable = false; | ||
| function readDisabledFromEnv(env) { | ||
| const raw = env.PLUR_DISABLE_EMBEDDINGS; | ||
| if (!raw) return null; | ||
| const normalized = raw.trim().toLowerCase(); | ||
| if (normalized === "1" || normalized === "true" || normalized === "yes") { | ||
| return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var"; | ||
| } | ||
| return null; | ||
| } | ||
| var ENV_DISABLED_REASON = readDisabledFromEnv(process.env); | ||
| var embeddingsDisabled = ENV_DISABLED_REASON !== null; | ||
| var disabledReason = ENV_DISABLED_REASON; | ||
| function embedderStatus() { | ||
| return { | ||
| available: !embeddingsDisabled && !transformersUnavailable, | ||
| loaded: embedPipeline !== null, | ||
| lastError: lastLoadError, | ||
| disabled: embeddingsDisabled, | ||
| disabledReason | ||
| }; | ||
| } | ||
| function setEmbeddingsEnabled(enabled, reason) { | ||
| embeddingsDisabled = !enabled; | ||
| disabledReason = enabled ? null : reason ?? "embeddings disabled by config"; | ||
| if (!enabled) { | ||
| embedPipeline = null; | ||
| } | ||
| } | ||
| function resetEmbedder() { | ||
| transformersUnavailable = false; | ||
| lastLoadError = null; | ||
| embedPipeline = null; | ||
| } | ||
| function _setCachedEmbedder(adapter) { | ||
| embedPipeline = adapter; | ||
| transformersUnavailable = false; | ||
| lastLoadError = null; | ||
| } | ||
| async function getEmbedder() { | ||
| if (embeddingsDisabled) return null; | ||
| if (embedPipeline) return embedPipeline; | ||
| try { | ||
| const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-TB252LRE.js"); | ||
| const adapter = getAdapter(resolveEmbedderName()); | ||
| embedPipeline = adapter; | ||
| transformersUnavailable = false; | ||
| lastLoadError = null; | ||
| return embedPipeline; | ||
| } catch (err) { | ||
| transformersUnavailable = true; | ||
| lastLoadError = err instanceof Error ? err.message : String(err); | ||
| return null; | ||
| } | ||
| } | ||
| async function embed(text, role) { | ||
| const embedder = await getEmbedder(); | ||
| if (!embedder) return null; | ||
| if (typeof embedder.embed === "function") { | ||
| let vector; | ||
| try { | ||
| vector = await embedder.embed(text, role); | ||
| } catch (err) { | ||
| transformersUnavailable = true; | ||
| lastLoadError = err instanceof Error ? err.message : String(err); | ||
| embedPipeline = null; | ||
| return null; | ||
| } | ||
| if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) { | ||
| throw new Error( | ||
| `Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.` | ||
| ); | ||
| } | ||
| return vector; | ||
| } | ||
| const result = await embedder(text, { pooling: "cls", normalize: true }); | ||
| return new Float32Array(result.data); | ||
| } | ||
| async function getActiveEmbedderMeta() { | ||
| const embedder = await getEmbedder(); | ||
| if (!embedder) return null; | ||
| if (typeof embedder.name === "string" && typeof embedder.dim === "number") { | ||
| return { name: embedder.name, dim: embedder.dim }; | ||
| } | ||
| return { name: "legacy-pipeline", dim: 0 }; | ||
| } | ||
| async function activeEmbedderDim() { | ||
| const meta = await getActiveEmbedderMeta(); | ||
| return meta && meta.dim > 0 ? meta.dim : null; | ||
| } | ||
| function cosineSimilarity(a, b) { | ||
| let dot = 0; | ||
| for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; | ||
| return dot; | ||
| } | ||
| var CACHE_VERSION = 1; | ||
| function emptyCache(meta) { | ||
| return { | ||
| meta: { | ||
| embedder_name: meta.name, | ||
| embedder_dim: meta.dim, | ||
| version: CACHE_VERSION | ||
| }, | ||
| entries: {} | ||
| }; | ||
| } | ||
| function loadCache(cachePath, active) { | ||
| if (!existsSync(cachePath)) return emptyCache(active); | ||
| try { | ||
| const raw = JSON.parse(readFileSync(cachePath, "utf8")); | ||
| if (!raw || typeof raw !== "object" || !raw.meta) { | ||
| logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`); | ||
| return emptyCache(active); | ||
| } | ||
| const meta = raw.meta; | ||
| if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) { | ||
| logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`); | ||
| return emptyCache(active); | ||
| } | ||
| const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {}; | ||
| return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries }; | ||
| } catch { | ||
| return emptyCache(active); | ||
| } | ||
| } | ||
| function saveCache(cachePath, cache) { | ||
| const dir = dirname(cachePath); | ||
| if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); | ||
| atomicWrite(cachePath, JSON.stringify(cache), { durable: false }); | ||
| } | ||
| function hashStatement(statement) { | ||
| return createHash("sha256").update(statement).digest("hex").slice(0, 16); | ||
| } | ||
| async function embeddingSearch(engrams, query, limit, storagePath) { | ||
| if (engrams.length === 0) return []; | ||
| const activeMeta = await getActiveEmbedderMeta(); | ||
| if (!activeMeta) return []; | ||
| const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json"; | ||
| const cache = loadCache(cachePath, activeMeta); | ||
| const queryEmbedding = await embed(query, "query"); | ||
| if (!queryEmbedding) { | ||
| return []; | ||
| } | ||
| const similarities = []; | ||
| for (const engram of engrams) { | ||
| const searchText = engramSearchText(engram); | ||
| const hash = hashStatement(searchText); | ||
| let engramEmbedding; | ||
| if (cache.entries[engram.id]?.hash === hash) { | ||
| engramEmbedding = new Float32Array(cache.entries[engram.id].embedding); | ||
| } else { | ||
| const emb = await embed(searchText); | ||
| if (!emb) return []; | ||
| engramEmbedding = emb; | ||
| cache.entries[engram.id] = { | ||
| hash, | ||
| embedding: Array.from(engramEmbedding) | ||
| }; | ||
| } | ||
| const score = cosineSimilarity(queryEmbedding, engramEmbedding); | ||
| similarities.push({ engram, score }); | ||
| } | ||
| saveCache(cachePath, cache); | ||
| similarities.sort((a, b) => b.score - a.score); | ||
| return similarities.slice(0, limit).map((s) => s.engram); | ||
| } | ||
| async function embeddingSearchWithScores(engrams, query, limit, storagePath) { | ||
| if (engrams.length === 0) return []; | ||
| const activeMeta = await getActiveEmbedderMeta(); | ||
| if (!activeMeta) return []; | ||
| const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json"; | ||
| const cache = loadCache(cachePath, activeMeta); | ||
| const queryEmbedding = await embed(query, "query"); | ||
| if (!queryEmbedding) { | ||
| return []; | ||
| } | ||
| const similarities = []; | ||
| for (const engram of engrams) { | ||
| const searchText = engramSearchText(engram); | ||
| const hash = hashStatement(searchText); | ||
| let engramEmbedding; | ||
| if (cache.entries[engram.id]?.hash === hash) { | ||
| engramEmbedding = new Float32Array(cache.entries[engram.id].embedding); | ||
| } else { | ||
| const emb = await embed(searchText); | ||
| if (!emb) return []; | ||
| engramEmbedding = emb; | ||
| cache.entries[engram.id] = { | ||
| hash, | ||
| embedding: Array.from(engramEmbedding) | ||
| }; | ||
| } | ||
| const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding); | ||
| const score = Math.max(0, Math.min(1, rawScore)); | ||
| similarities.push({ engram, score }); | ||
| } | ||
| saveCache(cachePath, cache); | ||
| similarities.sort((a, b) => b.score - a.score); | ||
| return similarities.slice(0, limit); | ||
| } | ||
| async function rebuildJsonCache(engrams, storagePath, opts) { | ||
| const activeMeta = await getActiveEmbedderMeta(); | ||
| if (!activeMeta) { | ||
| return { reembedded: 0, skipped: true, reason: "embedder unavailable" }; | ||
| } | ||
| const cachePath = join(storagePath, ".embeddings-cache.json"); | ||
| const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta); | ||
| let count = 0; | ||
| for (const engram of engrams) { | ||
| const searchText = engramSearchText(engram); | ||
| const hash = hashStatement(searchText); | ||
| if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue; | ||
| const vec = await embed(searchText); | ||
| if (!vec) { | ||
| return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" }; | ||
| } | ||
| cache.entries[engram.id] = { hash, embedding: Array.from(vec) }; | ||
| count++; | ||
| } | ||
| saveCache(cachePath, cache); | ||
| return { reembedded: count, skipped: false }; | ||
| } | ||
| export { | ||
| EMBED_DIM, | ||
| readDisabledFromEnv, | ||
| embedderStatus, | ||
| setEmbeddingsEnabled, | ||
| resetEmbedder, | ||
| _setCachedEmbedder, | ||
| embed, | ||
| activeEmbedderDim, | ||
| cosineSimilarity, | ||
| embeddingSearch, | ||
| embeddingSearchWithScores, | ||
| rebuildJsonCache | ||
| }; |
| import { | ||
| DEFAULT_EMBEDDER, | ||
| EMBEDDER_NAMES, | ||
| _resetEmbedderCache, | ||
| _resetResolveWarnings, | ||
| getEmbedder, | ||
| resolveEmbedderName | ||
| } from "./chunk-TWFYTLOE.js"; | ||
| import "./chunk-E4YVUWMJ.js"; | ||
| export { | ||
| DEFAULT_EMBEDDER, | ||
| EMBEDDER_NAMES, | ||
| _resetEmbedderCache, | ||
| _resetResolveWarnings, | ||
| getEmbedder, | ||
| resolveEmbedderName | ||
| }; |
| import { | ||
| EMBED_DIM, | ||
| _setCachedEmbedder, | ||
| activeEmbedderDim, | ||
| cosineSimilarity, | ||
| embed, | ||
| embedderStatus, | ||
| embeddingSearch, | ||
| embeddingSearchWithScores, | ||
| readDisabledFromEnv, | ||
| rebuildJsonCache, | ||
| resetEmbedder, | ||
| setEmbeddingsEnabled | ||
| } from "./chunk-W56Y5QPY.js"; | ||
| import "./chunk-SYCJM6JJ.js"; | ||
| import "./chunk-TXHLQGN3.js"; | ||
| import "./chunk-E4YVUWMJ.js"; | ||
| export { | ||
| EMBED_DIM, | ||
| _setCachedEmbedder, | ||
| activeEmbedderDim, | ||
| cosineSimilarity, | ||
| embed, | ||
| embedderStatus, | ||
| embeddingSearch, | ||
| embeddingSearchWithScores, | ||
| readDisabledFromEnv, | ||
| rebuildJsonCache, | ||
| resetEmbedder, | ||
| setEmbeddingsEnabled | ||
| }; |
| import { | ||
| MIN_TOKEN_LENGTH, | ||
| TOKENIZER_VERSION, | ||
| computeIdf, | ||
| embeddingContentHash, | ||
| engramSearchText, | ||
| extendCorpusStats, | ||
| ftsScore, | ||
| ftsTokenize, | ||
| hashEmbeddedText, | ||
| searchEngrams, | ||
| termMatches | ||
| } from "./chunk-SYCJM6JJ.js"; | ||
| export { | ||
| MIN_TOKEN_LENGTH, | ||
| TOKENIZER_VERSION, | ||
| computeIdf, | ||
| embeddingContentHash, | ||
| engramSearchText, | ||
| extendCorpusStats, | ||
| ftsScore, | ||
| ftsTokenize, | ||
| hashEmbeddedText, | ||
| searchEngrams, | ||
| termMatches | ||
| }; |
| import { | ||
| appendHistory, | ||
| buildDedupPrompt, | ||
| computeContentHash, | ||
| maybeDailyBackup, | ||
| parseDedupResponse | ||
| } from "./chunk-VU5HJWBU.js"; | ||
| import { | ||
| withAsyncLock | ||
| } from "./chunk-TXHLQGN3.js"; | ||
| import { | ||
| logger | ||
| } from "./chunk-E4YVUWMJ.js"; | ||
| // src/learn-async.ts | ||
| async function persistOne(deps, corpus, changed) { | ||
| if (deps.store.updateMany) { | ||
| await deps.store.updateMany([changed]); | ||
| deps.store.invalidate(); | ||
| return; | ||
| } | ||
| await deps.store.save(corpus); | ||
| } | ||
| async function withStoreLock(deps, fn) { | ||
| const guarded = async () => { | ||
| try { | ||
| maybeDailyBackup(deps.rootPath, deps.engramsPath); | ||
| } catch { | ||
| } | ||
| return await fn(); | ||
| }; | ||
| if (deps.store.withExclusiveAccess) return await deps.store.withExclusiveAccess(guarded); | ||
| return await withAsyncLock(deps.engramsPath, guarded); | ||
| } | ||
| function demoteIfSensitive(deps, engram, newStatement) { | ||
| const tags = Array.isArray(engram.tags) ? engram.tags.filter((t) => typeof t === "string") : []; | ||
| const scanText = tags.length ? `${newStatement} | ||
| ${tags.join(" ")}` : newStatement; | ||
| const offending = deps.offendingHitsForScope(scanText, engram.scope ?? "global"); | ||
| if (offending.length === 0) return; | ||
| const patterns = [...new Set(offending.map((h) => h.pattern))].join(", "); | ||
| logger.warning( | ||
| `[plur] sensitive content (${patterns}) held back from shared scope "${engram.scope}" \u2014 demoted to local/private so it is not written to a shared store. Re-scope deliberately if this is a false positive.` | ||
| ); | ||
| const from = engram.scope ?? "global"; | ||
| engram.scope = "local"; | ||
| engram.visibility = "private"; | ||
| engram.structured_data = { | ||
| ...engram.structured_data ?? {}, | ||
| _demoted: { from, to: "local", patterns } | ||
| }; | ||
| } | ||
| async function executeDedupDecision(deps, statement, context, decision, targetId) { | ||
| switch (decision) { | ||
| case "NOOP": { | ||
| if (targetId) { | ||
| const existing = await deps.getById(targetId); | ||
| if (existing) return { engram: existing, decision: "NOOP", existing_id: targetId }; | ||
| } | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| case "UPDATE": { | ||
| if (targetId) { | ||
| const existing = await deps.getById(targetId); | ||
| if (existing && existing.commitment !== "locked") { | ||
| const result = await withStoreLock(deps, async () => { | ||
| const engrams = await deps.store.load(); | ||
| const idx = engrams.findIndex((e) => e.id === targetId); | ||
| if (idx === -1) return null; | ||
| const updated = { ...engrams[idx] }; | ||
| updated.statement = statement; | ||
| updated.content_hash = computeContentHash(statement); | ||
| updated.engram_version = (updated.engram_version ?? 1) + 1; | ||
| updated.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10); | ||
| if (context?.tags) updated.tags = [.../* @__PURE__ */ new Set([...updated.tags, ...context.tags])]; | ||
| demoteIfSensitive(deps, updated, updated.statement); | ||
| engrams[idx] = updated; | ||
| await persistOne(deps, engrams, updated); | ||
| await deps.syncIndex(); | ||
| appendHistory(deps.rootPath, { | ||
| event: "engram_updated", | ||
| engram_id: targetId, | ||
| timestamp: (/* @__PURE__ */ new Date()).toISOString(), | ||
| data: { old_statement: existing.statement, new_statement: statement, reason: "LLM dedup UPDATE" } | ||
| }); | ||
| return { engram: updated, decision: "UPDATE", existing_id: targetId }; | ||
| }); | ||
| if (result) return result; | ||
| } | ||
| } | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| case "MERGE": { | ||
| if (targetId) { | ||
| const existing = await deps.getById(targetId); | ||
| if (existing && existing.commitment !== "locked") { | ||
| const result = await withStoreLock(deps, async () => { | ||
| const engrams = await deps.store.load(); | ||
| const idx = engrams.findIndex((e) => e.id === targetId); | ||
| if (idx === -1) return null; | ||
| const merged = { ...engrams[idx] }; | ||
| merged.statement = `${merged.statement} ${statement}`; | ||
| merged.content_hash = computeContentHash(merged.statement); | ||
| merged.engram_version = (merged.engram_version ?? 1) + 1; | ||
| merged.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10); | ||
| if (context?.tags) merged.tags = [.../* @__PURE__ */ new Set([...merged.tags, ...context.tags])]; | ||
| if (0.7 > merged.activation.retrieval_strength) merged.activation.retrieval_strength = 0.7; | ||
| demoteIfSensitive(deps, merged, merged.statement); | ||
| engrams[idx] = merged; | ||
| await persistOne(deps, engrams, merged); | ||
| await deps.syncIndex(); | ||
| appendHistory(deps.rootPath, { | ||
| event: "engram_merged", | ||
| engram_id: targetId, | ||
| timestamp: (/* @__PURE__ */ new Date()).toISOString(), | ||
| data: { merged_statement: statement, reason: "LLM dedup MERGE" } | ||
| }); | ||
| return { engram: merged, decision: "MERGE", existing_id: targetId }; | ||
| }); | ||
| if (result) return result; | ||
| } | ||
| } | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| case "ADD": | ||
| default: | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| } | ||
| async function learnAsync(deps, statement, context) { | ||
| const hashMatch = await deps.hashDedup(statement, context?.scope); | ||
| if (hashMatch) { | ||
| return { engram: hashMatch, decision: "NOOP", existing_id: hashMatch.id }; | ||
| } | ||
| const { enabled = true, threshold = 0.85, mode = "llm" } = deps.dedupConfig; | ||
| if (!enabled || mode === "off") { | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| let candidates = []; | ||
| try { | ||
| candidates = await deps.recallHybrid(statement, { limit: 5 }); | ||
| } catch { | ||
| candidates = await deps.recall(statement, { limit: 5 }); | ||
| } | ||
| if (candidates.length === 0) { | ||
| candidates = await deps.recall(statement, { limit: 5 }); | ||
| } | ||
| candidates = candidates.filter((c) => c.status === "active"); | ||
| if (context?.scope) { | ||
| candidates = candidates.filter((c) => c.scope === context.scope); | ||
| } | ||
| if (candidates.length === 0) { | ||
| return { engram: await deps.learn(statement, context), decision: "ADD" }; | ||
| } | ||
| const llm = context?.llm; | ||
| let decision = "ADD"; | ||
| let targetId = null; | ||
| if (mode === "llm" && llm && deps.isLlmAvailable()) { | ||
| try { | ||
| const prompt = buildDedupPrompt( | ||
| statement, | ||
| candidates.map((c) => ({ id: c.id, statement: c.statement, type: c.type, domain: c.domain })) | ||
| ); | ||
| const response = await llm(prompt); | ||
| const parsed = parseDedupResponse(response); | ||
| decision = parsed.decision; | ||
| targetId = parsed.target_id; | ||
| deps.recordLlmSuccess(); | ||
| } catch (err) { | ||
| logger.warning(`LLM dedup failed, falling back to cosine: ${err}`); | ||
| deps.recordLlmFailure(); | ||
| decision = "ADD"; | ||
| } | ||
| } | ||
| return executeDedupDecision(deps, statement, context, decision, targetId); | ||
| } | ||
| async function learnBatch(deps, statements, llm, opts = {}) { | ||
| const results = []; | ||
| const failures = []; | ||
| const stats = { added: 0, updated: 0, merged: 0, noops: 0, failed: 0 }; | ||
| const maxLlmCalls = opts.maxLlmCalls ?? 50; | ||
| let llmCallsUsed = 0; | ||
| let capWarned = false; | ||
| for (let i = 0; i < statements.length; i++) { | ||
| const { statement, context } = statements[i]; | ||
| const stmtLlm = context?.llm ?? llm; | ||
| let effectiveLlm = stmtLlm; | ||
| if (stmtLlm) { | ||
| if (llmCallsUsed >= maxLlmCalls) { | ||
| effectiveLlm = void 0; | ||
| if (!capWarned) { | ||
| logger.warning(`learnBatch: maxLlmCalls (${maxLlmCalls}) reached \u2014 remaining statements use cosine/ADD dedup`); | ||
| capWarned = true; | ||
| } | ||
| } else { | ||
| effectiveLlm = async (prompt) => { | ||
| llmCallsUsed++; | ||
| return stmtLlm(prompt); | ||
| }; | ||
| } | ||
| } | ||
| const ctx = { ...context, llm: effectiveLlm }; | ||
| try { | ||
| const result = await learnAsync(deps, statement, ctx); | ||
| results.push({ ...result, input_index: i }); | ||
| const key = result.decision.toLowerCase(); | ||
| if (key === "noop") stats.noops++; | ||
| else if (key === "update") stats.updated++; | ||
| else if (key === "merge") stats.merged++; | ||
| else stats.added++; | ||
| } catch (err) { | ||
| stats.failed++; | ||
| failures.push({ index: i, statement, error: err instanceof Error ? err.message : String(err) }); | ||
| logger.warning(`learnBatch: statement ${i} failed \u2014 ${err instanceof Error ? err.message : String(err)}`); | ||
| } | ||
| } | ||
| return { results, stats, failures }; | ||
| } | ||
| export { | ||
| learnAsync, | ||
| learnBatch | ||
| }; |
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 2 instances
URL strings
Supply chain riskPackage contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.
AI-detected potential code anomaly
Supply chain riskAI has identified unusual behaviors that may pose a security risk.
URL strings
Supply chain riskPackage contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.
1335377
8.42%30181
7.47%21
-16%59
1.72%+ Added
+ Added
- Removed
- Removed
Updated