New:Socket for Asana Is Now Available.Learn more
Get Started

@plur-ai/core

Package Overview
Dependencies
Maintainers
1
Versions
61
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@plur-ai/core - npm Package Compare versions

Comparing version
0.16.1
to
0.17.0
+170
dist/chunk-SKVT6ZGO.js
// src/fts.ts
import { createHash } from "crypto";
var STOP_WORDS = /* @__PURE__ */ new Set([
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"are",
"was",
"were",
"been",
"have",
"has",
"not",
"but",
"its",
"you",
"your",
"can",
"will",
"should",
"would",
"could",
"may",
"might"
]);
function ftsTokenize(text) {
return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
}
function engramSearchText(engram) {
const parts = [engram.statement];
if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
if (engram.entities) {
for (const e of engram.entities) {
parts.push(e.name);
if (e.type !== "other") parts.push(e.type);
}
}
if (engram.temporal) {
if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
}
if (engram.rationale) parts.push(engram.rationale);
if (engram.source) parts.push(engram.source);
if (engram.dual_coding) {
if (engram.dual_coding.example) parts.push(engram.dual_coding.example);
if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy);
}
if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) {
for (const a of engram.knowledge_anchors) {
if (a.snippet) parts.push(a.snippet);
}
}
return parts.join(" ");
}
function embeddingContentHash(engram) {
return hashEmbeddedText(engramSearchText(engram));
}
function hashEmbeddedText(text) {
return createHash("md5").update(text).digest("hex");
}
function termMatches(t, qt) {
return t.includes(qt) || qt.startsWith(t);
}
function computeIdf(engrams, queryTokens, stats) {
if (stats) {
if (stats.N === 0) return /* @__PURE__ */ new Map();
const idf2 = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
const df = stats.df.get(qt) ?? 0;
idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df))));
}
return idf2;
}
const N = engrams.length;
if (N === 0) return /* @__PURE__ */ new Map();
const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
const idf = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
let df = 0;
for (const termSet of engramTermSets) {
if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) {
df++;
}
}
idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
}
return idf;
}
function extendCorpusStats(stats, queryTokens, outsiders) {
if (outsiders.length === 0) return stats;
const termSets = [];
let totalLen = 0;
for (const e of outsiders) {
const terms = ftsTokenize(engramSearchText(e));
totalLen += terms.length;
termSets.push(new Set(terms));
}
const df = new Map(stats.df);
for (const qt of queryTokens) {
let added = 0;
for (const set of termSets) {
if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++;
}
if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added);
}
const N = stats.N + outsiders.length;
return {
N,
df,
avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0
};
}
var BM25_K1 = 1.2;
var BM25_B = 0.75;
function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
const allTerms = ftsTokenize(engramSearchText(engram));
if (queryTokens.length === 0) return 0;
const docLen = allTerms.length;
const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
let score = 0;
for (const qt of queryTokens) {
let effectiveIdf;
if (!idfWeights) {
effectiveIdf = 1;
} else if (hasNonZeroIdf) {
effectiveIdf = idfWeights.get(qt) ?? 0;
if (effectiveIdf === 0) continue;
} else {
effectiveIdf = 1;
}
let tf = 0;
for (const t of allTerms) {
if (termMatches(t, qt)) tf++;
}
if (tf === 0) continue;
const numerator = tf * (BM25_K1 + 1);
const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
score += effectiveIdf * (numerator / denominator);
}
return score;
}
function searchEngrams(engrams, query, limit = 20, stats) {
const queryTokens = ftsTokenize(query);
if (queryTokens.length === 0) return [];
const idfWeights = computeIdf(engrams, queryTokens, stats);
const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0);
if (scored.length === 0) {
scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0);
}
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
}
export {
ftsTokenize,
engramSearchText,
embeddingContentHash,
hashEmbeddedText,
termMatches,
computeIdf,
extendCorpusStats,
ftsScore,
searchEngrams
};
// src/scope-util.ts
var SHARED_SCOPE_PREFIXES = ["group:", "project:", "space:", "team:", "org:", "public"];
function isSharedScope(scope) {
const s = scope.toLowerCase();
return SHARED_SCOPE_PREFIXES.some(
(p) => p === "public" ? s === "public" || s.startsWith("public:") || s.startsWith("public/") : s.startsWith(p)
);
}
function isPersonalScope(scope) {
return !isSharedScope(scope);
}
function isScopeWithin(scope, queryScope) {
return scope === queryScope || scope.startsWith(queryScope + ":") || scope.startsWith(queryScope + "/");
}
function makeVisibilityPredicate(scopeFilter, grants) {
return (engramScope) => isScopeWithin(engramScope, scopeFilter) || isPersonalScope(engramScope) || grants !== void 0 && grants.some((g) => isScopeWithin(engramScope, g));
}
function scopeAllowFilter(scopes) {
if (scopes === void 0) return () => true;
const allowed = new Set(scopes);
return (scope) => allowed.has(scope);
}
// src/async-mutex.ts
var AsyncMutex = class {
queue = Promise.resolve();
/** Number of runs queued or executing. Drives KeyedAsyncMutex eviction. */
depth = 0;
/** True when nothing is queued or running. */
get idle() {
return this.depth === 0;
}
async run(fn) {
let release;
const wait = new Promise((res) => {
release = res;
});
const prev = this.queue;
this.queue = prev.then(() => wait);
this.depth++;
await prev;
try {
return await fn();
} finally {
this.depth--;
release();
}
}
};
var KeyedAsyncMutex = class {
mutexes = /* @__PURE__ */ new Map();
/** Number of keys with work queued or running. Test/diagnostic seam. */
get size() {
return this.mutexes.size;
}
async run(key, fn) {
let mutex = this.mutexes.get(key);
if (!mutex) {
mutex = new AsyncMutex();
this.mutexes.set(key, mutex);
}
try {
return await mutex.run(fn);
} finally {
if (mutex.idle && this.mutexes.get(key) === mutex) this.mutexes.delete(key);
}
}
};
// src/store/async-lock.ts
import { writeFile, unlink, stat, readFile, rename, open } from "fs/promises";
import { constants } from "fs";
import { hostname } from "os";
import * as path from "path";
var DEFAULT_STALE_THRESHOLD = 6e4;
var DEFAULT_ACQUIRE_TIMEOUT = 18e4;
var processLocks = new KeyedAsyncMutex();
function sleep(ms) {
return new Promise((res) => setTimeout(res, ms));
}
var tokenCounter = 0;
function makeToken() {
return `${hostname()}:${process.pid}:${Date.now()}:${tokenCounter++}`;
}
function holderIsAlive(token) {
const parts = token.split(":");
if (parts.length < 2) return void 0;
const [host, pidRaw] = parts;
if (host !== hostname()) return void 0;
const pid = Number(pidRaw);
if (!Number.isInteger(pid) || pid <= 0) return void 0;
try {
process.kill(pid, 0);
return true;
} catch (err) {
if (err?.code === "EPERM") return true;
return false;
}
}
async function withFileLock(filePath, fn, options) {
const lockPath = filePath + ".lock";
const baseDelay = options?.baseDelay ?? 100;
const staleThreshold = options?.staleThreshold ?? DEFAULT_STALE_THRESHOLD;
const acquireTimeout = options?.acquireTimeout ?? Math.max(
DEFAULT_ACQUIRE_TIMEOUT,
// A caller that raises staleThreshold must not thereby make waiters give up
// before it — the inversion F9 was about.
Math.ceil(staleThreshold * 1.5)
);
const maxRetries = options?.maxRetries;
const token = makeToken();
const start = Date.now();
let acquired = false;
let lastHolder = "";
for (let attempt = 0; ; attempt++) {
if (attempt > 0) {
const elapsed = Date.now() - start;
const outOfRetries = maxRetries !== void 0 && attempt > maxRetries;
if (elapsed >= acquireTimeout || outOfRetries) {
throw new Error(
`Failed to acquire lock on ${filePath} after ${attempt} attempt(s) / ${Math.round(elapsed / 1e3)}s${lastHolder ? ` (held by ${lastHolder})` : ""}.
A live holder is waited for, never stolen from \u2014 stealing a lock from a process that is still writing corrupts the store. If the holder is genuinely stuck, stop it and remove ${lockPath}.`
);
}
}
try {
await writeFile(lockPath, token, { flag: constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL });
acquired = true;
break;
} catch (err) {
if (err.code !== "EEXIST") throw err;
let abandoned = false;
let holder = "";
try {
const [s, contents] = await Promise.all([
stat(lockPath),
readFile(lockPath, "utf8").catch(() => "")
]);
holder = contents.trim();
const alive = holderIsAlive(holder);
if (alive === false) abandoned = true;
else if (alive === void 0 && Date.now() - s.mtimeMs > staleThreshold) abandoned = true;
} catch {
continue;
}
if (abandoned) {
await stealLock(lockPath, holder);
continue;
}
lastHolder = holder;
await sleep(Math.min(baseDelay * Math.pow(2, attempt), 5e3));
}
}
try {
return await fn();
} finally {
if (acquired) await releaseIfOurs(lockPath, token);
}
}
async function stealLock(lockPath, expected) {
const claim = `${lockPath}.steal.${makeToken().replace(/[^\w.-]/g, "_")}`;
try {
await rename(lockPath, claim);
} catch {
return;
}
try {
const current = (await readFile(claim, "utf8")).trim();
if (current === expected) {
await unlink(claim);
return;
}
try {
const fd = await open(lockPath, "wx");
try {
await fd.writeFile(current);
} finally {
await fd.close();
}
} catch {
}
await unlink(claim).catch(() => {
});
} catch {
await unlink(claim).catch(() => {
});
}
}
async function releaseIfOurs(lockPath, token) {
try {
const current = (await readFile(lockPath, "utf8")).trim();
if (current !== token) return;
await unlink(lockPath);
} catch {
}
}
async function withAsyncLock(filePath, fn, options) {
return processLocks.run(path.resolve(filePath), () => withFileLock(filePath, fn, options));
}
// src/sync.ts
import { execFileSync } from "child_process";
import { existsSync, readFileSync, writeFileSync, renameSync, mkdirSync, unlinkSync, statSync, readdirSync, openSync, closeSync, fsyncSync, chmodSync } from "fs";
import { join, dirname, relative } from "path";
import * as yaml from "js-yaml";
var GITIGNORE = `# PLUR \u2014 secrets (machine-local, NEVER synced)
config.yaml
secrets.yaml
agent-keystore.json
*.token
# PLUR \u2014 derived/cache files (regenerated automatically)
embeddings/
.embeddings-cache.json
*.db
*.sqlite
store.pglite/
exchange/
# PLUR \u2014 local backups (#799). Machine-local by design: a snapshot is a
# whole-corpus copy INCLUDING scope:local engrams, so pushing one would leak
# exactly the engrams the scope strip exists to hold back.
backups/
*.superseded-*
`;
var SYNC_PATHS = ["engrams.yaml", "episodes.yaml", "candidates.yaml", "tensions.yaml", "packs", ".gitignore"];
var SECRET_PATHS = ["config.yaml", "secrets.yaml", "agent-keystore.json"];
var PACK_ALLOW_NAMES = ["SKILL.md", "engrams.yaml", "INTEGRITY", "metadata.json"];
function git(args, cwd) {
return execFileSync("git", args, { cwd, encoding: "utf8", timeout: 3e4 }).trim();
}
function gitSafe(args, cwd) {
try {
return git(args, cwd);
} catch {
return null;
}
}
function isGitRepo(root) {
return existsSync(join(root, ".git"));
}
function hasGitCli() {
try {
execFileSync("git", ["--version"], { encoding: "utf8", timeout: 5e3 });
return true;
} catch {
return false;
}
}
function getRemote(root) {
return gitSafe(["remote", "get-url", "origin"], root);
}
function isDirty(root) {
const status = gitSafe(["status", "--porcelain"], root);
return status !== null && status.length > 0;
}
function countDiff(root, direction) {
const tracking = gitSafe(["rev-parse", "--abbrev-ref", "@{u}"], root);
if (!tracking) return 0;
const flag = direction === "ahead" ? "--left-only" : "--right-only";
const count = gitSafe(["rev-list", flag, "--count", "HEAD...@{u}"], root);
return count ? parseInt(count, 10) : 0;
}
function getSyncStatus(root) {
if (!isGitRepo(root)) {
return { initialized: false, remote: null, dirty: false, branch: null, ahead: 0, behind: 0 };
}
const branch = gitSafe(["rev-parse", "--abbrev-ref", "HEAD"], root);
const remote = getRemote(root);
if (remote) gitSafe(["fetch", "origin", "--quiet"], root);
return {
initialized: true,
remote,
dirty: isDirty(root),
branch,
ahead: countDiff(root, "ahead"),
behind: countDiff(root, "behind")
};
}
function stageStoreFiles(root) {
assertNoUnmergedStoreFiles(root);
for (const secret of SECRET_PATHS) {
gitSafe(["rm", "--cached", "--ignore-unmatch", "--quiet", "--", secret], root);
}
const present = SYNC_PATHS.filter((p) => p !== "packs" && existsSync(join(root, p)));
const pathspecs = [...present, ...packStorePaths(root)];
if (pathspecs.length > 0) {
git(["add", "-A", "-f", "--", ...pathspecs], root);
}
const staged = gitSafe(["diff", "--cached", "--name-only"], root);
return staged ? staged.split("\n").filter(Boolean).length : 0;
}
function assertNoUnmergedStoreFiles(root) {
const unmerged = gitSafe(["diff", "--name-only", "--diff-filter=U"], root);
if (!unmerged) return;
const conflicted = unmerged.split("\n").map((f) => f.trim()).filter(Boolean).filter((f) => SYNC_PATHS.some((p) => f === p || f.startsWith(`${p}/`)));
if (conflicted.length === 0) return;
throw new Error(
`[plur] refusing to sync: ${conflicted.join(", ")} ${conflicted.length === 1 ? "is" : "are"} unmerged.
Staging an unmerged file would mark the conflict "resolved" with its markers still in it, and commit (and push) that as your engram store.
Resolve the conflict first. If a previous sync left an autostash behind, your complete copy may be in 'git stash list' \u2014 check it BEFORE running 'git stash drop' or 'git reset --hard'.`
);
}
function packStorePaths(root) {
const packsDir = join(root, "packs");
if (!existsSync(packsDir)) return [];
const allow = new Set(PACK_ALLOW_NAMES);
const paths = /* @__PURE__ */ new Set();
const stack = [packsDir];
while (stack.length > 0) {
const dir = stack.pop();
for (const ent of readdirSync(dir, { withFileTypes: true })) {
const full = join(dir, ent.name);
if (ent.isDirectory()) stack.push(full);
else if (allow.has(ent.name)) paths.add(relative(root, full));
}
}
const tracked = gitSafe(["ls-files", "--", "packs"], root);
if (tracked) {
for (const f of tracked.split("\n").filter(Boolean)) {
if (allow.has(f.split("/").pop() ?? "")) paths.add(f);
}
}
return [...paths];
}
var YAML_DUMP_OPTS = { lineWidth: 120, noRefs: true, quotingType: '"' };
var SIBLING_STRIP_FILES = ["episodes.yaml", "candidates.yaml", "tensions.yaml"];
var ENGRAM_ID_TOKEN = /\b(?:ENG|ABS|META)-[A-Za-z0-9-]+/g;
var SIBLING_DUMP_OPTS = { lineWidth: 120, noRefs: true };
var SyncStoreUnreadableError = class extends Error {
constructor(filePath) {
super(
`[plur] refusing to sync: cannot parse ${filePath}.
The scope filter that keeps private and scope:local engrams off the remote is derived from this file. With it unreadable, PLUR cannot tell which engrams must NOT be pushed, so continuing would commit the file verbatim \u2014 publishing private engrams and any conflict markers along with them.
Common cause: a merge conflict in engrams.yaml \u2014 look for <<<<<<< markers. If a previous sync left an autostash behind, your complete copy may be in 'git stash list' \u2014 check it BEFORE running 'git stash drop' or 'git reset --hard'.
Fix the file, then retry.`
);
this.filePath = filePath;
this.name = "SyncStoreUnreadableError";
}
filePath;
};
function readEngramList(root) {
const path2 = join(root, "engrams.yaml");
if (!existsSync(path2)) return null;
let raw;
try {
raw = yaml.load(readFileSync(path2, "utf8"));
} catch {
throw new SyncStoreUnreadableError(path2);
}
if (Array.isArray(raw)) return { raw, list: raw };
if (raw && typeof raw === "object" && Array.isArray(raw.engrams)) {
return { raw, list: raw.engrams };
}
throw new SyncStoreUnreadableError(path2);
}
function pushKeep(remoteType) {
if (remoteType === "shared") {
return (e) => isSharedScope(e?.scope ?? "") && (e?.visibility ?? "private") !== "private";
}
return (e) => e?.scope !== "local";
}
function sharedPushIds(root) {
const parsed = readEngramList(root);
if (!parsed) return /* @__PURE__ */ new Set();
const keep = pushKeep("shared");
return new Set(
parsed.list.filter(keep).map((e) => String(e?.id ?? "")).filter(Boolean)
);
}
function readSiblingList(root, file) {
const path2 = join(root, file);
if (!existsSync(path2)) return null;
let raw;
try {
raw = yaml.load(readFileSync(path2, "utf8"));
} catch {
throw new SyncStoreUnreadableError(path2);
}
if (raw == null) throw new SyncStoreUnreadableError(path2);
if (!Array.isArray(raw)) throw new SyncStoreUnreadableError(path2);
return raw;
}
function siblingKeep(pushedIds) {
return (record) => {
const text = JSON.stringify(record) ?? "";
const refs = text.match(ENGRAM_ID_TOKEN);
if (!refs) return true;
return refs.every((id) => pushedIds.has(id));
};
}
function droppedSiblingCount(root) {
const keep = siblingKeep(sharedPushIds(root));
let dropped = 0;
for (const file of SIBLING_STRIP_FILES) {
const records = readSiblingList(root, file);
if (!records) continue;
dropped += records.filter((r) => !keep(r)).length;
}
return dropped;
}
function stripWarning(root, remoteType) {
const parsed = readEngramList(root);
if (remoteType === "shared") {
const strippedEngrams = parsed ? parsed.list.filter((e) => !pushKeep("shared")(e)).length : 0;
const strippedSiblings = droppedSiblingCount(root);
if (strippedEngrams === 0 && strippedSiblings === 0) return void 0;
const parts2 = [];
if (strippedEngrams > 0) parts2.push(`${strippedEngrams} personal-scope or private-visibility engram(s)`);
if (strippedSiblings > 0) parts2.push(`${strippedSiblings} episode/candidate/tension record(s) derived from non-pushed engrams`);
return `Shared remote: pushed only shared-scope, non-private engrams \u2014 ${parts2.join(" and ")} stayed local.`;
}
if (!parsed) return void 0;
const privateCount = parsed.list.filter(
(e) => e?.scope !== "local" && (e?.visibility ?? "private") === "private"
).length;
const localCount = parsed.list.filter((e) => e?.scope === "local").length;
const parts = [];
if (localCount > 0) {
parts.push(
`${localCount} scope:local engram(s) are NOT pushed and are NOT backed up by sync` + (localCount === parsed.list.length ? " \u2014 this remote backs up nothing" : "")
);
}
if (privateCount > 0) {
parts.push(`${privateCount} private-visibility engram(s) ARE pushed \u2014 use a private git remote`);
}
if (parts.length === 0) return void 0;
return `Note: ${parts.join(". ")}. For a team remote, set sync.remote_type: shared to exclude private engrams too.`;
}
function stageBlob(root, relPath, content) {
const hash = execFileSync("git", ["hash-object", "-w", "--stdin"], {
cwd: root,
input: content,
encoding: "utf8",
timeout: 3e4
}).trim();
git(["update-index", "--cacheinfo", `100644,${hash},${relPath}`], root);
}
function stageStripped(root, remoteType) {
stageStrippedEngrams(root, remoteType);
stageStrippedSiblings(root, remoteType);
}
function stageStrippedEngrams(root, remoteType) {
const parsed = readEngramList(root);
if (!parsed) return;
const { raw, list } = parsed;
const keep = pushKeep(remoteType);
const filtered = list.filter(keep);
if (filtered.length === list.length) return;
const out = Array.isArray(raw) ? yaml.dump(filtered, YAML_DUMP_OPTS) : yaml.dump({ ...raw, engrams: filtered }, YAML_DUMP_OPTS);
stageBlob(root, "engrams.yaml", out);
}
function stageStrippedSiblings(root, remoteType) {
if (remoteType !== "shared") return;
const keep = siblingKeep(sharedPushIds(root));
for (const file of SIBLING_STRIP_FILES) {
const records = readSiblingList(root, file);
if (!records) continue;
const kept = records.filter(keep);
if (kept.length === records.length) continue;
stageBlob(root, file, yaml.dump(kept, SIBLING_DUMP_OPTS));
}
}
function initRepo(root, remoteType) {
git(["init"], root);
atomicWrite(join(root, ".gitignore"), GITIGNORE);
stageStoreFiles(root);
stageStripped(root, remoteType);
git(["commit", "-m", "Initial PLUR engram store"], root);
}
function commitChanges(root, remoteType) {
const filesChanged = stageStoreFiles(root);
if (filesChanged === 0) return 0;
stageStripped(root, remoteType);
const diff = gitSafe(["diff", "--cached", "--shortstat"], root);
if (!diff || diff.length === 0) return 0;
const now = (/* @__PURE__ */ new Date()).toISOString().slice(0, 19).replace("T", " ");
git(["commit", "-m", `plur sync ${now}`], root);
const match = diff.match(/(\d+) file/);
return match ? parseInt(match[1], 10) : filesChanged;
}
function hasConflictMarkers(root) {
const result = gitSafe(["grep", "-l", "<<<<<<<"], root);
return result !== null && result.length > 0;
}
function pullRebase(root, remoteType) {
const branch = gitSafe(["rev-parse", "--abbrev-ref", "HEAD"], root) || "main";
const result = gitSafe(["pull", "--rebase", "origin", branch], root);
if (result !== null) return true;
gitSafe(["rebase", "--abort"], root);
const mergeResult = gitSafe(["pull", "origin", branch, "--no-edit"], root);
if (mergeResult !== null) return true;
if (hasConflictMarkers(root)) {
gitSafe(["merge", "--abort"], root);
throw new Error("Sync conflict: YAML files have merge conflicts that require manual resolution. Your local changes are preserved.");
}
stageStoreFiles(root);
stageStripped(root, remoteType);
const committed = gitSafe(["commit", "-m", "plur sync: merge conflict resolved (kept both)"], root);
if (committed === null) return false;
return countDiff(root, "behind") === 0;
}
function sync(root, remote, options) {
if (!hasGitCli()) {
throw new Error("git is not installed. Install git to enable sync.");
}
const remoteType = options?.remoteType ?? "personal";
if (!isGitRepo(root)) {
initRepo(root, remoteType);
if (remote) {
git(["remote", "add", "origin", remote], root);
const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], root);
git(["push", "-u", "origin", branch], root);
return { action: "initialized", message: `Initialized and pushed to ${remote}`, remote, files_changed: 0, warning: stripWarning(root, remoteType) };
}
return {
action: "initialized",
message: "Initialized local git repo. Call plur.sync with remote to enable cross-device sync.",
remote: null,
files_changed: 0
};
}
const existingRemote = getRemote(root);
if (remote && !existingRemote) {
git(["remote", "add", "origin", remote], root);
const filesChanged2 = commitChanges(root, remoteType);
const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], root);
git(["push", "-u", "origin", branch], root);
return { action: "synced", message: `Remote added and pushed to ${remote}`, remote, files_changed: filesChanged2, warning: stripWarning(root, remoteType) };
}
if (!existingRemote) {
const filesChanged2 = commitChanges(root, remoteType);
if (filesChanged2 === 0) {
return { action: "up-to-date", message: 'No changes to commit. Add a remote with await plur.sync({ remote: "..." }) to enable cross-device sync.', remote: null, files_changed: 0 };
}
return { action: "committed", message: `Committed ${filesChanged2} file(s) locally.`, remote: null, files_changed: filesChanged2 };
}
const filesChanged = commitChanges(root, remoteType);
gitSafe(["fetch", "origin", "--quiet"], root);
const behind = countDiff(root, "behind");
const aheadBefore = countDiff(root, "ahead");
let pullFailed = false;
if (behind > 0) {
pullFailed = !pullRebase(root, remoteType);
}
const behindAfter = behind > 0 ? countDiff(root, "behind") : 0;
const pulled = behind - behindAfter;
let pushError = null;
const aheadAfter = countDiff(root, "ahead");
if (aheadAfter > 0) {
try {
git(["push", "origin"], root);
} catch (err) {
pushError = (err.message || "").trim() || "git push failed";
}
}
if (filesChanged === 0 && behind === 0 && aheadBefore === 0) {
return { action: "up-to-date", message: "Already in sync.", remote: existingRemote, files_changed: 0, warning: stripWarning(root, remoteType) };
}
const parts = [];
if (filesChanged > 0) parts.push(`${filesChanged} file(s) committed`);
if (pulled > 0) parts.push(`pulled ${pulled} remote commit(s)`);
if (pullFailed || behindAfter > 0) {
parts.push(`NOT pulled \u2014 still ${behindAfter} commit(s) behind the remote; resolve locally and retry`);
}
if (aheadAfter === 0 && aheadBefore > 0) parts.push("pushed");
if (pushError) parts.push("NOT pushed \u2014 the commit is local only");
return {
action: "synced",
message: `Synced. ${parts.join(", ")}.`,
remote: existingRemote,
files_changed: filesChanged,
warning: stripWarning(root, remoteType),
...pushError ? { push_error: pushError } : {}
};
}
function stealLockSync(lockPath, expected, token) {
const claim = `${lockPath}.steal.${token.replace(/[^\w.-]/g, "_")}`;
try {
renameSync(lockPath, claim);
} catch {
return;
}
try {
const current = readFileSync(claim, "utf8").trim();
if (current === expected) {
unlinkSync(claim);
return;
}
try {
const fd = openSync(lockPath, "wx");
try {
writeFileSync(fd, current);
} finally {
closeSync(fd);
}
} catch {
}
try {
unlinkSync(claim);
} catch {
}
} catch {
try {
unlinkSync(claim);
} catch {
}
}
}
function withLock(filePath, fn, options) {
const lockPath = filePath + ".lock";
const maxRetries = options?.maxRetries ?? 5;
const baseDelay = options?.baseDelay ?? 100;
const staleThreshold = options?.staleThreshold ?? DEFAULT_STALE_THRESHOLD;
const token = makeToken();
let acquired = false;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
writeFileSync(lockPath, token, { flag: "wx" });
acquired = true;
break;
} catch (err) {
if (err.code !== "EEXIST") throw err;
try {
const stat2 = statSync(lockPath);
const holder = readFileSync(lockPath, "utf8").trim();
const alive = holderIsAlive(holder);
const steal = alive === false || alive === void 0 && Date.now() - stat2.mtimeMs > staleThreshold;
if (steal) {
stealLockSync(lockPath, holder, token);
continue;
}
} catch {
continue;
}
if (attempt === maxRetries) {
throw new Error(`Failed to acquire lock on ${filePath} after ${maxRetries} retries`);
}
const delay = baseDelay * Math.pow(2, attempt);
const end = Date.now() + delay;
while (Date.now() < end) {
}
}
}
if (!acquired) {
throw new Error(
`Failed to acquire lock on ${filePath} after ${maxRetries} retries (contended throughout)`
);
}
try {
return fn();
} finally {
try {
if (readFileSync(lockPath, "utf8").trim() === token) unlinkSync(lockPath);
} catch {
}
}
}
var CONFIG_FILE_MODE = 384;
var tmpCounter = 0;
function atomicWrite(filePath, content, opts = {}) {
const durable = opts.durable !== false;
const dir = dirname(filePath);
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
const tmp = `${filePath}.${process.pid}.${tmpCounter++}.tmp`;
let destMode = opts.mode;
try {
if (existsSync(filePath)) destMode = statSync(filePath).mode & 4095;
} catch {
}
try {
if (durable) {
const fd = openSync(tmp, "w");
try {
writeFileSync(fd, content);
fsyncSync(fd);
} finally {
closeSync(fd);
}
} else {
writeFileSync(tmp, content);
}
if (destMode !== void 0) {
try {
chmodSync(tmp, destMode);
} catch {
}
}
renameSync(tmp, filePath);
if (durable) fsyncDir(dir);
} catch (err) {
try {
unlinkSync(tmp);
} catch {
}
throw err;
}
}
function fsyncDir(dir) {
let fd;
try {
fd = openSync(dir, "r");
fsyncSync(fd);
} catch {
} finally {
if (fd !== void 0) {
try {
closeSync(fd);
} catch {
}
}
}
}
export {
SHARED_SCOPE_PREFIXES,
isSharedScope,
isPersonalScope,
isScopeWithin,
makeVisibilityPredicate,
scopeAllowFilter,
AsyncMutex,
KeyedAsyncMutex,
withAsyncLock,
getSyncStatus,
sync,
withLock,
CONFIG_FILE_MODE,
atomicWrite,
fsyncDir
};
import {
engramSearchText
} from "./chunk-SKVT6ZGO.js";
import {
atomicWrite
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/embeddings.ts
import { existsSync, readFileSync, mkdirSync } from "fs";
import { join, dirname } from "path";
import { createHash } from "crypto";
var EMBED_DIM = 384;
var embedPipeline = null;
var lastLoadError = null;
var transformersUnavailable = false;
function readDisabledFromEnv(env) {
const raw = env.PLUR_DISABLE_EMBEDDINGS;
if (!raw) return null;
const normalized = raw.trim().toLowerCase();
if (normalized === "1" || normalized === "true" || normalized === "yes") {
return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var";
}
return null;
}
var ENV_DISABLED_REASON = readDisabledFromEnv(process.env);
var embeddingsDisabled = ENV_DISABLED_REASON !== null;
var disabledReason = ENV_DISABLED_REASON;
function embedderStatus() {
return {
available: !embeddingsDisabled && !transformersUnavailable,
loaded: embedPipeline !== null,
lastError: lastLoadError,
disabled: embeddingsDisabled,
disabledReason
};
}
function setEmbeddingsEnabled(enabled, reason) {
embeddingsDisabled = !enabled;
disabledReason = enabled ? null : reason ?? "embeddings disabled by config";
if (!enabled) {
embedPipeline = null;
}
}
function resetEmbedder() {
transformersUnavailable = false;
lastLoadError = null;
embedPipeline = null;
}
function _setCachedEmbedder(adapter) {
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
}
async function getEmbedder() {
if (embeddingsDisabled) return null;
if (embedPipeline) return embedPipeline;
try {
const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-TB252LRE.js");
const adapter = getAdapter(resolveEmbedderName());
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
return embedPipeline;
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
return null;
}
}
async function embed(text, role) {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.embed === "function") {
let vector;
try {
vector = await embedder.embed(text, role);
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
embedPipeline = null;
return null;
}
if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) {
throw new Error(
`Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.`
);
}
return vector;
}
const result = await embedder(text, { pooling: "cls", normalize: true });
return new Float32Array(result.data);
}
async function getActiveEmbedderMeta() {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.name === "string" && typeof embedder.dim === "number") {
return { name: embedder.name, dim: embedder.dim };
}
return { name: "legacy-pipeline", dim: 0 };
}
async function activeEmbedderDim() {
const meta = await getActiveEmbedderMeta();
return meta && meta.dim > 0 ? meta.dim : null;
}
function cosineSimilarity(a, b) {
let dot = 0;
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
return dot;
}
var CACHE_VERSION = 1;
function emptyCache(meta) {
return {
meta: {
embedder_name: meta.name,
embedder_dim: meta.dim,
version: CACHE_VERSION
},
entries: {}
};
}
function loadCache(cachePath, active) {
if (!existsSync(cachePath)) return emptyCache(active);
try {
const raw = JSON.parse(readFileSync(cachePath, "utf8"));
if (!raw || typeof raw !== "object" || !raw.meta) {
logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`);
return emptyCache(active);
}
const meta = raw.meta;
if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) {
logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`);
return emptyCache(active);
}
const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {};
return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries };
} catch {
return emptyCache(active);
}
}
function saveCache(cachePath, cache) {
const dir = dirname(cachePath);
if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true });
atomicWrite(cachePath, JSON.stringify(cache), { durable: false });
}
function hashStatement(statement) {
return createHash("sha256").update(statement).digest("hex").slice(0, 16);
}
async function embeddingSearch(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const score = cosineSimilarity(queryEmbedding, engramEmbedding);
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit).map((s) => s.engram);
}
async function embeddingSearchWithScores(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding);
const score = Math.max(0, Math.min(1, rawScore));
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit);
}
async function rebuildJsonCache(engrams, storagePath, opts) {
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) {
return { reembedded: 0, skipped: true, reason: "embedder unavailable" };
}
const cachePath = join(storagePath, ".embeddings-cache.json");
const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta);
let count = 0;
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue;
const vec = await embed(searchText);
if (!vec) {
return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" };
}
cache.entries[engram.id] = { hash, embedding: Array.from(vec) };
count++;
}
saveCache(cachePath, cache);
return { reembedded: count, skipped: false };
}
export {
EMBED_DIM,
readDisabledFromEnv,
embedderStatus,
setEmbeddingsEnabled,
resetEmbedder,
_setCachedEmbedder,
embed,
activeEmbedderDim,
cosineSimilarity,
embeddingSearch,
embeddingSearchWithScores,
rebuildJsonCache
};
import {
atomicWrite,
withLock
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/schemas/engram.ts
import { z } from "zod";
var ActivationSchema = z.object({
retrieval_strength: z.number().min(0).max(1),
storage_strength: z.number().min(0).max(1),
frequency: z.number().int().min(0),
last_accessed: z.string().describe("Date or ISO 8601 timestamp of last access.")
}).describe("ACT-R activation parameters driving decay and ranking. STABLE.");
var KnowledgeTypeSchema = z.object({
memory_class: z.enum(["semantic", "episodic", "procedural", "metacognitive"]),
cognitive_level: z.enum(["remember", "understand", "apply", "analyze", "evaluate", "create"]).describe("Bloom's taxonomy level.")
});
var KnowledgeAnchorSchema = z.object({
path: z.string().describe("Path to a grounding document/file."),
relevance: z.enum(["primary", "supporting", "example"]).default("supporting"),
snippet: z.string().max(200).optional(),
snippet_extracted_at: z.string().optional()
});
var AssociationSchema = z.object({
target_type: z.enum(["engram", "document"]),
target: z.string().describe("ID or path of the association target."),
strength: z.number().min(0).max(0.95),
type: z.enum(["semantic", "temporal", "causal", "co_accessed"]),
updated_at: z.string().optional()
});
var DualCodingSchema = z.object({
example: z.string().optional(),
analogy: z.string().optional()
}).describe("Worked example and/or analogy (dual coding). At least one of example or analogy MUST be provided (enforced at runtime by the Zod .refine below).").refine(
(d) => d.example || d.analogy,
"At least one of example or analogy must be provided"
);
var RelationsSchema = z.object({
broader: z.array(z.string()).default([]),
narrower: z.array(z.string()).default([]),
related: z.array(z.string()).default([]),
conflicts: z.array(z.string()).default([]),
/** IDs of engrams this one intentionally replaces (#240). An intentional
* update is not a tension — the scanner skips supersedes-linked pairs. */
supersedes: z.array(z.string()).default([]),
/** Reverse edge of `supersedes` (#240) — IDs of engrams that replace this one. */
superseded_by: z.array(z.string()).default([])
}).describe("Typed graph edges between engram IDs.");
var ProvenanceSchema = z.object({
origin: z.string(),
chain: z.array(z.string()).default([]),
signature: z.string().nullable().default(null).describe("RESERVED. Detached signature over the engram. Algorithm and canonicalization not yet specified \u2014 see ENGRAM-STANDARD-v1.md \xA77."),
license: z.string().default("cc-by-sa-4.0")
}).describe("Origin and signing chain. STABLE for origin/chain/license; signature is RESERVED (see ENGRAM-STANDARD-v1.md \xA77).");
var FeedbackSignalsSchema = z.object({
positive: z.number().int().default(0),
negative: z.number().int().default(0),
neutral: z.number().int().default(0)
});
var EntityRefSchema = z.object({
name: z.string(),
type: z.enum([
"person",
"organization",
"technology",
"concept",
"project",
"tool",
"place",
"event",
"standard",
"other"
]),
uri: z.string().url().optional()
});
var TemporalSchema = z.object({
learned_at: z.string(),
valid_from: z.string().optional(),
valid_until: z.string().optional(),
ingested_at: z.string().optional()
}).describe("Bi-temporal anchoring (Zep-inspired). When is this knowledge true?");
var UsageStatsSchema = z.object({
injections: z.number().int().default(0),
hits: z.number().int().default(0),
misses: z.number().int().default(0),
last_hit_at: z.string().optional()
});
var EpisodicFieldsSchema = z.object({
emotional_weight: z.number().int().min(1).max(10).default(5),
confidence: z.number().int().min(1).max(10).default(5),
trigger_context: z.string().optional(),
journal_ref: z.string().optional()
});
var PreviousVersionRefSchema = z.object({
event_id: z.string(),
changed_at: z.string()
});
var ExchangeMetadataSchema = z.object({
fitness_score: z.number().min(0).max(1).optional(),
environmental_diversity: z.number().int().default(0),
adoption_count: z.number().int().default(0),
contradiction_rate: z.number().min(0).max(1).default(0)
});
var SerendipitySchema = z.object({
unexpectedness: z.number().min(0).max(1),
relevance: z.number().min(0).max(1),
score: z.number().min(0).max(1)
});
var InsightFateSchema = z.enum([
"surfaced",
// shown in a briefing; no downstream action yet
"promoted",
// became a durable engram / zettel
"cited",
// referenced in later journal/work
"tasked",
// converted to a GTD task
"dismissed",
// user/LLM rejected it
"expired"
// decayed out of the buffer unused
]);
var InsightFieldSchema = z.object({
/** Which memory-stream operation produced this insight. Nightly arc:
* `distill` (episode→insight synthesis) → `consolidate` (convergent gist
* abstraction over the buffer) → `dream` (divergent REM-style recombination —
* speculative, never auto-promoted). `connect`/`emerge`/`drift` are on-demand lenses. */
operation: z.enum(["distill", "consolidate", "dream", "connect", "emerge", "drift"]),
synthesized_at: z.string(),
/** Anti-hallucination grounding. Cited source notes live in the parent engram's
* `knowledge_anchors[]`; this flags whether the claim was verified against those
* snippets. `ungrounded` = couldn't cite sources → quarantined (`candidate`, never
* surfaced). `speculative` = a `dream`: its recombined INPUTS are cited but its
* CONCLUSION is an explicit hypothesis — surfaced only as inspiration, and (per the
* promote-requires-grounding refine below) it must be re-grounded to `verified`
* before it can be promoted to a durable engram. */
grounding: z.enum(["verified", "unverified", "ungrounded", "speculative"]).default("unverified"),
/** The episode-log slice this insight was distilled from (evidence trail). */
source_episode_ids: z.array(z.string()).default([]),
/** Distinct objective for connect/emerge/dream insights. */
serendipity: SerendipitySchema.optional(),
fate: InsightFateSchema.default("surfaced"),
/** Engram id / zettel path / task id the insight became, if acted upon. */
fate_ref: z.string().optional(),
fate_at: z.string().optional(),
/** How many briefings have surfaced this insight (acted-upon-rate denominator). */
surfaced_count: z.number().int().min(0).default(0)
}).refine(
// Promote-requires-grounding (user rule 2026-06-15): a dream is inspiration, not
// fact. A speculative/ungrounded insight can only become a durable promotion once
// it has been re-grounded in reality (grounding=verified).
(i) => i.fate !== "promoted" || i.grounding === "verified",
{ message: "A promoted insight must be grounded (grounding=verified); speculative dreams cannot be promoted until re-grounded.", path: ["grounding"] }
);
var ExtractionProvenanceSchema = z.object({
confidence: z.number().min(0).max(1).optional().describe("0-1 classifier confidence at extraction time. Frozen at write; distinct from feedback-derived computeConfidence() and from episodic.confidence."),
source_commit: z.string().optional().describe("Git SHA of the source repository at extraction time (reproducibility)."),
extractor_version: z.string().optional().describe("Version of the extracting CLI/tool (schema-migration handle). Complementary to the pack-level capsule producer field (#61).")
}).passthrough().describe("ETL extraction provenance convention carried in structured_data.extraction (#463). Not wired into EngramSchema.");
function getExtractionProvenance(engram) {
const extraction = engram.structured_data?.["extraction"];
if (extraction === void 0 || extraction === null) return null;
const parsed = ExtractionProvenanceSchema.safeParse(extraction);
return parsed.success ? parsed.data : null;
}
var EngramSchema = z.object({
// Identity
id: z.string().regex(/^(ENG|ABS|META)-[A-Za-z0-9-]+$/).describe("Unique identifier. Class prefix ENG (concrete engram), ABS (abstraction), or META (meta-engram). Canonical concrete form: ENG-YYYY-MMDD-NNN; store-namespaced form: ENG-{PREFIX}-YYYY-MMDD-NNN."),
version: z.number().int().min(1).default(2).describe("Schema-shape generation of this engram object (currently 2). Distinct from engram_version, which tracks content evolution."),
// 'active' and 'retired' are the two states any current code path assigns
// (retire via forget/dedup/supersede). 'dormant' and 'candidate' are NOT
// assigned by any code today: 'dormant' was only ever set by the batchDecay
// pass removed in #563 (decay is now a read-time property, not a materialized
// status), and 'candidate' is reserved. They are kept in the enum so stores
// written before #563 that persisted status:'dormant' still load, and so the
// status filter accepts them; do not remove without a data migration.
status: z.enum(["active", "dormant", "retired", "candidate"]).describe("Lifecycle state. Assigned values today are active/retired; dormant/candidate are legacy/reserved (see note above)."),
consolidated: z.boolean().default(false).describe("Whether this engram has been through consolidation (sleep-like batch reprocessing)."),
type: z.enum(["behavioral", "terminological", "procedural", "architectural"]).describe("Top-level classification of the knowledge."),
scope: z.string().describe("Hierarchical namespace, e.g. 'global', 'project:my-app', 'group:plur/test'. Free-form string; ':' separates scope kind from path."),
visibility: z.enum(["private", "public", "template"]).default("private").describe("Sharing posture. 'private' engrams MUST NOT be exported in packs."),
// Content
statement: z.string().min(1).describe("The assertion itself \u2014 the load-bearing content of the engram."),
rationale: z.string().optional().describe("Why this is true / why it matters."),
contraindications: z.array(z.string()).optional().describe("Conditions under which the statement does NOT apply."),
// Lineage
source: z.string().optional().describe("Free-text origin (session, document, conversation)."),
source_patterns: z.array(z.string()).optional().describe("Pattern IDs that contributed to this engram."),
derivation_count: z.number().int().min(0).default(1).describe("How many derivation steps produced this engram."),
pack: z.string().nullable().default(null).describe("Name of the pack this engram belongs to, or null."),
abstract: z.string().nullable().default(null).describe("ID of an ABS- abstraction this engram instantiates, or null."),
derived_from: z.string().nullable().default(null).describe("ID of the engram this was derived from, or null."),
// Classification
knowledge_type: KnowledgeTypeSchema.optional(),
domain: z.string().optional().describe("Dotted domain path, e.g. 'dev/testing' or 'plur.session'."),
tags: z.array(z.string()).default([]).describe("Free-form tags used for matching and retrieval."),
// Activation (ACT-R model)
activation: ActivationSchema.default({
retrieval_strength: 0.7,
storage_strength: 1,
frequency: 0,
last_accessed: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
}),
// Relations & grounding
relations: RelationsSchema.optional(),
associations: z.array(AssociationSchema).default([]),
knowledge_anchors: z.array(KnowledgeAnchorSchema).default([]),
dual_coding: DualCodingSchema.optional(),
// Provenance
provenance: ProvenanceSchema.optional(),
// Feedback
feedback_signals: FeedbackSignalsSchema.default({ positive: 0, negative: 0, neutral: 0 }),
// === NEW OPTIONAL FIELDS (v2.1) ===
/** Typed entity references extracted from statement. Enables graph queries. */
entities: z.array(EntityRefSchema).optional().describe("Typed entity references extracted from statement. Enables graph queries."),
/** Temporal validity window. When is this knowledge true? */
temporal: TemporalSchema.optional(),
/** Automatic usage tracking. Injections, hits, misses. */
usage: UsageStatsSchema.optional(),
/** Episodic context: emotional weight, confidence, trigger. */
episodic: EpisodicFieldsSchema.optional(),
/** Exchange marketplace metadata: fitness, adoption, diversity. */
exchange: ExchangeMetadataSchema.optional(),
/** Extensible key-value data for domain-specific fields. */
structured_data: z.record(z.string(), z.unknown()).optional().describe("Extensible key-value data for domain-specific fields."),
/** Memory-stream insight provenance (metacognition Phase 1). Orthogonal to
* `type`. Present iff this engram was synthesized by the metacognition memory
* stream; the episodic insight buffer is the set of engrams where this is set. */
insight: InsightFieldSchema.optional(),
/** Polarity classification: 'do' for directives, 'dont' for prohibitions, null for unclassified. */
polarity: z.enum(["do", "dont"]).nullable().default(null).describe("'do' for directives, 'dont' for prohibitions, null for unclassified."),
// === SP1: Memory Intelligence fields ===
content_hash: z.string().optional().describe("Hash of normalized statement content, used for dedup."),
commitment: z.enum(["exploring", "leaning", "decided", "locked"]).optional().describe("Commitment level of the asserted knowledge."),
locked_at: z.string().optional().describe("Timestamp when commitment reached 'locked'."),
locked_reason: z.string().optional().describe("Why this engram was locked."),
// === SP1: Reference counting (issue #107) ===
/** Number of write attempts that resolved to this engram.
* Incremented on every hash-dedup hit; decremented by forget().
* Engram physically retires only when this reaches 0. */
reference_count: z.number().int().min(0).default(1).describe("Number of write attempts that resolved to this engram (same-scope re-learns). Engram retires only when this reaches 0."),
/** Provenance of each write attempt. One entry per write (including the
* first). Migrated old engrams without this field start with []. */
sources: z.array(z.object({
scope: z.string(),
session_id: z.string().nullable().default(null),
stored_at: z.string().describe("ISO 8601 timestamp of this write.")
})).default([]).describe("Provenance of each write attempt; one entry per write."),
// === SP1: Cross-scope recurrence (issue #176) ===
/** Number of times this engram's content was re-learned at a DIFFERENT
* scope than the original. Triggers auto-broadening + commitment
* escalation when threshold is crossed. Distinct from reference_count
* (which counts re-learns in the SAME scope) — recurrence_count is
* evidence of universal applicability, not just repetition. */
recurrence_count: z.number().int().min(0).default(0).describe("Number of times this content was re-learned at a DIFFERENT scope than the original. Evidence of universal applicability."),
// === SP2: History & Evolution fields ===
engram_version: z.number().int().min(1).default(1).describe("Content-evolution version (incremented when the statement materially changes)."),
previous_version_ref: PreviousVersionRefSchema.optional(),
episode_ids: z.array(z.string()).default([]).describe("IDs of episodes (raw conversational events) that produced or reinforced this engram."),
// === SP3: Retrieval & Injection fields ===
summary: z.string().max(80).optional().describe("Short (<=80 char) injection-friendly summary."),
/**
* Always-load flag. Pinned engrams bypass the term-hits gate in scoreEngram
* and are eligible for injection on every session start, regardless of
* keyword overlap with the user's task. Use sparingly: meta-rules,
* cross-cutting safety conventions, and core operating principles only.
* Pinned engrams still respect the token budget — they bypass per-pack and
* per-domain fairness caps in fillTokenBudget so always-load behavior is
* honored even if a single pack contributes many.
*/
pinned: z.boolean().optional().describe("Always-load flag. Pinned engrams bypass the keyword-relevance gate and are eligible for injection every session. Use sparingly.")
});
var EngramSchemaPassthrough = EngramSchema.passthrough();
// src/backup.ts
import * as fs from "fs";
import * as path from "path";
import { createHash } from "crypto";
import * as yaml from "js-yaml";
var BACKUP_DIR = "backups";
var KEEP_DAILY = 7;
var KEEP_WEEKLY = 4;
var SHRINK_TOLERANCE = 0.1;
function statePath(root) {
return path.join(root, BACKUP_DIR, ".state.json");
}
function readState(root) {
try {
return JSON.parse(fs.readFileSync(statePath(root), "utf8"));
} catch {
return {};
}
}
function writeState(root, state) {
const p = statePath(root);
fs.mkdirSync(path.dirname(p), { recursive: true });
fs.writeFileSync(p, JSON.stringify(state, null, 2) + "\n", "utf8");
}
function sha256(content) {
return createHash("sha256").update(content).digest("hex");
}
function validateStore(filePath, lastGoodCount) {
const failures = [];
const reasons = [];
let raw;
try {
raw = fs.readFileSync(filePath);
} catch (err) {
return { ok: false, failures: ["unreadable"], reasons: [`cannot read ${filePath}: ${err}`], count: null };
}
if (raw.length === 0) {
return { ok: false, failures: ["empty"], reasons: ["file is 0 bytes"], count: null };
}
if (!raw.toString("utf8").endsWith("\n")) {
failures.push("truncated");
reasons.push("file does not end with a newline \u2014 PLUR's writer always emits one, so this looks cut short");
}
let doc;
try {
doc = yaml.load(raw.toString("utf8"));
} catch (err) {
return { ok: false, failures: ["unparseable"], reasons: [`YAML parse failed: ${err}`], count: null };
}
if (doc == null || typeof doc !== "object" || Array.isArray(doc) || !Array.isArray(doc.engrams)) {
return {
ok: false,
failures: ["not-a-store"],
reasons: ["parsed, but is not a mapping with an `engrams` list"],
count: null
};
}
const entries = doc.engrams;
const count = entries.length;
let invalid = 0;
const ids = /* @__PURE__ */ new Set();
let duplicateIds = 0;
let missingIds = 0;
for (const entry of entries) {
if (!EngramSchemaPassthrough.safeParse(entry).success) invalid++;
const id = entry?.id;
if (typeof id !== "string" || id.length === 0) missingIds++;
else if (ids.has(id)) duplicateIds++;
else ids.add(id);
}
if (invalid > 0) {
failures.push("invalid-entries");
reasons.push(`${invalid} entry/entries fail schema validation`);
}
if (missingIds > 0) {
failures.push("missing-ids");
reasons.push(`${missingIds} entry/entries have no id`);
}
if (duplicateIds > 0) {
failures.push("duplicate-ids");
reasons.push(`${duplicateIds} duplicate id(s)`);
}
if (typeof lastGoodCount === "number" && lastGoodCount > 0) {
const floor = lastGoodCount * (1 - SHRINK_TOLERANCE);
if (count < floor) {
failures.push("shrunk");
reasons.push(
`holds ${count} engram(s) but the last good snapshot held ${lastGoodCount} \u2014 a drop this large is how a truncation looks`
);
}
}
return { ok: failures.length === 0, failures, reasons, count };
}
function todayStamp(now) {
return now.toISOString().slice(0, 10);
}
function snapshotPath(root, stamp) {
return path.join(root, BACKUP_DIR, `engrams-${stamp}.yaml`);
}
var doneThisProcess = /* @__PURE__ */ new Set();
function maybeDailyBackup(root, storePath, now = /* @__PURE__ */ new Date()) {
const key = `${root}\0${todayStamp(now)}`;
if (doneThisProcess.has(key)) return { taken: false, skipped: "already-today" };
try {
if (!fs.existsSync(storePath)) {
doneThisProcess.add(key);
return { taken: false, skipped: "no-store" };
}
const state = readState(root);
const stamp = todayStamp(now);
if (state.last_backup_date === stamp) {
doneThisProcess.add(key);
return { taken: false, skipped: "already-today" };
}
const existing = listBackups(root);
const strongest = existing.reduce(
(max, b) => typeof b.count === "number" && (max === void 0 || b.count > max) ? b.count : max,
void 0
);
const baseline = state.last_good_count ?? strongest;
const validity = validateStore(storePath, baseline);
if (!validity.ok) {
logger.warning(
`[plur:backup] refusing to snapshot ${storePath} \u2014 ${validity.reasons.join("; ")}. Your last good backup is unchanged. Run 'plur doctor' to inspect.`
);
return { taken: false, skipped: "invalid", validity };
}
const bytes = fs.readFileSync(storePath);
const dest = snapshotPath(root, stamp);
const sameDay = existing.find((b) => b.stamp === stamp);
if (sameDay && typeof sameDay.count === "number" && (validity.count ?? 0) < sameDay.count) {
logger.warning(
`[plur:backup] keeping today's existing snapshot (${sameDay.count} engrams) \u2014 the live store holds ${validity.count}, and replacing a stronger snapshot with a weaker one would discard the better copy. Run 'plur restore --list' to inspect.`
);
doneThisProcess.add(key);
return { taken: false, skipped: "invalid", validity };
}
fs.mkdirSync(path.dirname(dest), { recursive: true });
writeFileDurable(dest, bytes);
writeFileDurable(
`${dest}.sha256`,
Buffer.from(
`${sha256(bytes)} ${path.basename(dest)}
${validity.count} engrams
taken_at ${now.toISOString()}
`,
"utf8"
)
);
doneThisProcess.add(key);
writeState(root, {
last_backup_date: stamp,
last_good_count: validity.count ?? void 0,
last_good_sha256: sha256(bytes)
});
rotate(root, now);
return { taken: true, path: dest, validity };
} catch (err) {
logger.warning(`[plur:backup] snapshot failed (the write itself was unaffected): ${err}`);
return { taken: false, skipped: "invalid" };
}
}
function flushFileAt(filePath) {
let fd;
try {
fd = fs.openSync(filePath, "r+");
fs.fsyncSync(fd);
} catch {
} finally {
if (fd !== void 0) {
try {
fs.closeSync(fd);
} catch {
}
}
}
}
function writeFileDurable(dest, bytes) {
const fd = fs.openSync(dest, "w");
try {
fs.writeFileSync(fd, bytes);
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
}
function listBackups(root) {
const dir = path.join(root, BACKUP_DIR);
if (!fs.existsSync(dir)) return [];
const out = [];
for (const name of fs.readdirSync(dir)) {
const m = name.match(/^engrams-(\d{4}-\d{2}-\d{2})\.yaml$/);
if (!m) continue;
const full = path.join(dir, name);
const entry = { path: full, stamp: m[1], size: fs.statSync(full).size };
try {
const sidecar = fs.readFileSync(`${full}.sha256`, "utf8");
entry.sha256 = sidecar.split(/\s+/)[0];
const cm = sidecar.match(/(\d+) engrams/);
if (cm) entry.count = parseInt(cm[1], 10);
const tm = sidecar.match(/taken_at (\S+)/);
if (tm) entry.takenAt = tm[1];
} catch {
}
out.push(entry);
}
return out.sort((a, b) => a.stamp < b.stamp ? 1 : -1);
}
function rotate(root, now) {
const all = listBackups(root);
if (all.length <= KEEP_DAILY) return;
const keep = /* @__PURE__ */ new Set();
for (const b of all.slice(0, KEEP_DAILY)) keep.add(b.path);
const weeksSeen = /* @__PURE__ */ new Set();
for (const b of all.slice(KEEP_DAILY)) {
const week = isoWeek(/* @__PURE__ */ new Date(`${b.stamp}T00:00:00Z`));
if (weeksSeen.has(week)) continue;
weeksSeen.add(week);
if (weeksSeen.size <= KEEP_WEEKLY) keep.add(b.path);
}
for (const b of all) {
if (keep.has(b.path)) continue;
try {
fs.unlinkSync(b.path);
fs.unlinkSync(`${b.path}.sha256`);
} catch {
}
}
void now;
}
function isoWeek(d) {
const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
const day = t.getUTCDay() || 7;
t.setUTCDate(t.getUTCDate() + 4 - day);
const yearStart = new Date(Date.UTC(t.getUTCFullYear(), 0, 1));
const week = Math.ceil(((t.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
return `${t.getUTCFullYear()}-W${week}`;
}
function planRestore(root, storePath, stamp) {
const all = listBackups(root);
if (all.length === 0) throw new Error(`[plur] no backups found in ${path.join(root, BACKUP_DIR)}`);
const backup = stamp ? all.find((b) => b.stamp === stamp) : all[0];
if (!backup) throw new Error(`[plur] no backup for ${stamp}. Available: ${all.map((b) => b.stamp).join(", ")}`);
const bytes = fs.readFileSync(backup.path);
const actualSha256 = sha256(bytes);
const integrityOk = backup.sha256 === void 0 ? false : backup.sha256 === actualSha256;
const validity = validateStore(backup.path);
const backupIds = new Set(idsIn(backup.path));
const currentIds = idsIn(storePath);
const wouldLose = currentIds.filter((id) => !backupIds.has(id));
return {
backup,
validity,
actualSha256,
integrityOk,
wouldLose,
// Compare against the snapshot's INSTANT where we have it. Falling back to
// the end of its day is the conservative direction when a sidecar predates
// this field: it under-reports rather than inventing losses.
unrecoverable: idsCreatedAfter(root, backup.takenAt ?? `${backup.stamp}T23:59:59.999Z`).filter((id) => !backupIds.has(id))
};
}
function idsIn(filePath) {
try {
const doc = yaml.load(fs.readFileSync(filePath, "utf8"));
if (!doc || !Array.isArray(doc.engrams)) return [];
return doc.engrams.map((e) => e?.id).filter((id) => typeof id === "string");
} catch {
return [];
}
}
function idsCreatedAfter(root, since) {
const dir = path.join(root, "history");
if (!fs.existsSync(dir)) return [];
const ids = [];
for (const name of fs.readdirSync(dir)) {
if (!name.endsWith(".jsonl")) continue;
let lines;
try {
lines = fs.readFileSync(path.join(dir, name), "utf8").split("\n");
} catch {
continue;
}
for (const line of lines) {
if (!line.trim()) continue;
try {
const ev = JSON.parse(line);
if (typeof ev?.timestamp !== "string" || ev.timestamp <= since) continue;
if (typeof ev?.engram_id === "string") ids.push(ev.engram_id);
} catch {
}
}
}
return [...new Set(ids)];
}
function restoreBackup(root, storePath, opts = {}) {
let plan;
const superseded = `${storePath}.superseded-${Date.now()}`;
withLock(storePath, () => {
plan = planRestore(root, storePath, opts.stamp);
if (!opts.force) {
const problems = [];
if (!plan.validity.ok) problems.push(...plan.validity.reasons);
if (!plan.integrityOk) {
problems.push(
plan.backup.sha256 === void 0 ? "no sha256 sidecar \u2014 cannot verify the backup is intact" : "sha256 does not match the sidecar \u2014 the backup itself is damaged"
);
}
if (problems.length > 0) {
throw new Error(
`[plur] refusing to restore ${plan.backup.path}: ${problems.join("; ")}.
Restoring is a whole-corpus overwrite; doing it from a backup that does not verify would replace a damaged store with a differently damaged one.
Pass force to override if you have inspected the file yourself.`
);
}
}
if (fs.existsSync(storePath)) {
fs.copyFileSync(storePath, superseded);
flushFileAt(superseded);
}
atomicWrite(storePath, fs.readFileSync(plan.backup.path, "utf8"));
});
if (plan.wouldLose.length > 0) {
logger.warning(
`[plur:restore] ${plan.wouldLose.length} engram(s) present before the restore are not in this backup: ${plan.wouldLose.slice(0, 10).join(", ")}${plan.wouldLose.length > 10 ? ", \u2026" : ""}. The pre-restore store was kept at ${superseded}.`
);
}
if (plan.unrecoverable.length > 0) {
logger.warning(
`[plur:restore] history records ${plan.unrecoverable.length} engram(s) created after this backup that it does not contain: ${plan.unrecoverable.slice(0, 10).join(", ")}${plan.unrecoverable.length > 10 ? ", \u2026" : ""}.`
);
}
return { ...plan, restored: true, supersededPath: superseded };
}
// src/history.ts
import * as fs2 from "fs";
import { join as join2 } from "path";
import { createHash as createHash2 } from "crypto";
function appendHistory(root, event) {
const historyDir = join2(root, "history");
if (!fs2.existsSync(historyDir)) {
fs2.mkdirSync(historyDir, { recursive: true });
}
const date = event.timestamp.slice(0, 7);
const filePath = join2(historyDir, `${date}.jsonl`);
const line = JSON.stringify(event) + "\n";
const fd = fs2.openSync(filePath, "a");
try {
fs2.writeSync(fd, line);
try {
fs2.fsyncSync(fd);
} catch {
}
} finally {
fs2.closeSync(fd);
}
}
function readHistory(root, yearMonth) {
const filePath = join2(root, "history", `${yearMonth}.jsonl`);
if (!fs2.existsSync(filePath)) return [];
const content = fs2.readFileSync(filePath, "utf8");
const lines = content.split("\n").filter((l) => l.trim().length > 0);
const events = [];
for (const line of lines) {
try {
events.push(JSON.parse(line));
} catch {
}
}
return events;
}
function listHistoryMonths(root) {
const historyDir = join2(root, "history");
if (!fs2.existsSync(historyDir)) return [];
return fs2.readdirSync(historyDir).filter((f) => f.endsWith(".jsonl")).map((f) => f.replace(".jsonl", "")).sort();
}
function readHistoryForEngram(root, engramId) {
const months = listHistoryMonths(root);
const events = [];
for (const month of months) {
const monthEvents = readHistory(root, month);
for (const event of monthEvents) {
if (event.engram_id === engramId) {
events.push(event);
}
}
}
return events;
}
var _PROC_SALT = (process.pid % 1296).toString(36).padStart(2, "0");
var _evtSeq = 0;
var _injSeq = 0;
function generateEventId() {
return `EVT-${Date.now()}-${_PROC_SALT}${(_evtSeq++).toString(36).padStart(4, "0")}`;
}
function generateInjectionId() {
return `INJ-${Date.now()}-${_PROC_SALT}${(_injSeq++).toString(36).padStart(4, "0")}`;
}
function computeQueryHash(task) {
const normalized = task.toLowerCase().replace(/\s+/g, " ").trim();
return createHash2("sha256").update(normalized).digest("hex").slice(0, 16);
}
function findLatestInjectionFor(root, engramId, maxMonths = 2) {
const now = /* @__PURE__ */ new Date();
const allowed = /* @__PURE__ */ new Set();
for (let i = 0; i < maxMonths; i++) {
const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1));
allowed.add(d.toISOString().slice(0, 7));
}
const months = listHistoryMonths(root).filter((m) => allowed.has(m)).reverse();
for (const month of months) {
let latest = null;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const ids = event.data.ids;
if (!Array.isArray(ids) || !ids.includes(engramId)) continue;
if (!latest || event.timestamp > latest.timestamp) latest = event;
}
if (latest) return { injection_id: latest.engram_id, timestamp: latest.timestamp };
}
return null;
}
var INJECTION_SOURCES = /* @__PURE__ */ new Set([
"session_start",
"inject",
"hook",
"unknown"
]);
var ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
function readCoInjections(root, months) {
const events = [];
let skipped = 0;
const wanted = months ? new Set(months) : null;
for (const month of listHistoryMonths(root)) {
if (wanted && !wanted.has(month)) continue;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const raw = event.data;
if (!Array.isArray(raw.ids) || typeof raw.query_hash !== "string") {
skipped++;
continue;
}
if (typeof event.timestamp !== "string" || !ISO_TIMESTAMP.test(event.timestamp)) {
skipped++;
continue;
}
const ids = raw.ids.filter((id) => typeof id === "string" && id.length > 0);
if (ids.length !== raw.ids.length) skipped++;
const data = { ids, query_hash: raw.query_hash };
if (typeof raw.tokens_used === "number" && Number.isFinite(raw.tokens_used)) {
data.tokens_used = raw.tokens_used;
}
if (raw.source !== void 0) {
data.source = INJECTION_SOURCES.has(raw.source) ? raw.source : "unknown";
}
if (typeof raw.scope === "string") data.scope = raw.scope;
if (typeof raw.session_id === "string") data.session_id = raw.session_id;
events.push({ injection_id: event.engram_id, timestamp: event.timestamp, data });
}
}
events.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
return { events, skipped };
}
function countInjectionEvents(root) {
const counts = {
co_injection: 0,
injection_outcome: 0,
outcome_positive: 0,
outcome_negative: 0
};
for (const month of listHistoryMonths(root)) {
for (const event of readHistory(root, month)) {
if (event.event === "co_injection") {
counts.co_injection++;
} else if (event.event === "injection_outcome") {
counts.injection_outcome++;
if (event.data.signal === "positive") counts.outcome_positive++;
else if (event.data.signal === "negative") counts.outcome_negative++;
}
}
}
return counts;
}
// src/content-hash.ts
import { createHash as createHash3 } from "crypto";
function normalizeStatement(statement) {
return statement.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
}
function computeContentHash(statement) {
const normalized = normalizeStatement(statement);
return createHash3("sha256").update(normalized).digest("hex");
}
// src/dedup.ts
function buildDedupPrompt(newStatement, candidates) {
const candidateList = candidates.map(
(c, i) => `${i + 1}. [${c.id}] (${c.type}${c.domain ? ", domain: " + c.domain : ""})
"${c.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare a new memory statement against existing ones.
NEW STATEMENT:
"${newStatement}"
EXISTING ENGRAMS:
${candidateList}
For each existing engram, answer:
1. RELATIONSHIP: Is the new statement a DUPLICATE (same meaning), EVOLUTION (updated version of same knowledge), COMPLEMENTARY (related but different angle), or UNRELATED?
2. RICHNESS: Does the new statement contain more specific, actionable information than the existing one? (yes/no)
Then give your OVERALL DECISION (exactly one):
- NOOP: New statement is an exact duplicate of an existing engram. Return the ID.
- UPDATE: New statement is an evolution with MORE information. Return the ID to update.
- MERGE: New statement and an existing one are complementary \u2014 combining them preserves both. Return the ID to merge with.
- ADD: New statement is genuinely new knowledge.
Respond in this exact format:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID if UPDATE/MERGE/NOOP, or "none" if ADD>
REASON: <one sentence explanation>`;
}
function buildBatchDedupPrompt(statements, existingEngrams) {
const stmtList = statements.map((s, i) => `${i + 1}. "${s}"`).join("\n");
const engramList = existingEngrams.map(
(e, i) => `${i + 1}. [${e.id}] (${e.type}${e.domain ? ", domain: " + e.domain : ""})
"${e.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare NEW statements against existing engrams.
NEW STATEMENTS:
${stmtList}
EXISTING ENGRAMS:
${engramList}
For each NEW statement, decide:
- NOOP: Exact duplicate of an existing engram.
- UPDATE: Evolution with more info than existing.
- MERGE: Complementary with existing \u2014 combine.
- ADD: Genuinely new knowledge.
Respond with one block per new statement:
STATEMENT_1:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID or "none">
STATEMENT_2:
...`;
}
function parseDedupResponse(response) {
const decisionMatch = response.match(/DECISION:\s*(ADD|UPDATE|MERGE|NOOP)/i);
const targetMatch = response.match(/TARGET:\s*([^\n]+)/i);
const reasonMatch = response.match(/REASON:\s*([^\n]+)/i);
const decision = decisionMatch?.[1]?.toUpperCase() ?? "ADD";
const targetRaw = targetMatch?.[1]?.trim() ?? "none";
const target_id = targetRaw === "none" ? null : targetRaw.replace(/[^A-Za-z0-9-]/g, "");
const reason = reasonMatch?.[1]?.trim() ?? "";
return { decision, target_id, reason };
}
export {
ExtractionProvenanceSchema,
getExtractionProvenance,
EngramSchemaPassthrough,
BACKUP_DIR,
validateStore,
maybeDailyBackup,
listBackups,
planRestore,
restoreBackup,
appendHistory,
readHistory,
listHistoryMonths,
readHistoryForEngram,
generateEventId,
generateInjectionId,
computeQueryHash,
findLatestInjectionFor,
readCoInjections,
countInjectionEvents,
normalizeStatement,
computeContentHash,
buildDedupPrompt,
buildBatchDedupPrompt,
parseDedupResponse
};
import {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
} from "./chunk-UND3VZDP.js";
import "./chunk-SKVT6ZGO.js";
import "./chunk-TXHLQGN3.js";
import "./chunk-E4YVUWMJ.js";
export {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
};
import {
computeIdf,
embeddingContentHash,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
hashEmbeddedText,
searchEngrams,
termMatches
} from "./chunk-SKVT6ZGO.js";
export {
computeIdf,
embeddingContentHash,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
hashEmbeddedText,
searchEngrams,
termMatches
};
import {
appendHistory,
buildDedupPrompt,
computeContentHash,
maybeDailyBackup,
parseDedupResponse
} from "./chunk-VU5HJWBU.js";
import {
withAsyncLock
} from "./chunk-TXHLQGN3.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/learn-async.ts
async function persistOne(deps, corpus, changed) {
if (deps.store.updateMany) {
await deps.store.updateMany([changed]);
deps.store.invalidate();
return;
}
await deps.store.save(corpus);
}
async function withStoreLock(deps, fn) {
const guarded = async () => {
try {
maybeDailyBackup(deps.rootPath, deps.engramsPath);
} catch {
}
return await fn();
};
if (deps.store.withExclusiveAccess) return await deps.store.withExclusiveAccess(guarded);
return await withAsyncLock(deps.engramsPath, guarded);
}
function demoteIfSensitive(deps, engram, newStatement) {
const tags = Array.isArray(engram.tags) ? engram.tags.filter((t) => typeof t === "string") : [];
const scanText = tags.length ? `${newStatement}
${tags.join(" ")}` : newStatement;
const offending = deps.offendingHitsForScope(scanText, engram.scope ?? "global");
if (offending.length === 0) return;
const patterns = [...new Set(offending.map((h) => h.pattern))].join(", ");
logger.warning(
`[plur] sensitive content (${patterns}) held back from shared scope "${engram.scope}" \u2014 demoted to local/private so it is not written to a shared store. Re-scope deliberately if this is a false positive.`
);
const from = engram.scope ?? "global";
engram.scope = "local";
engram.visibility = "private";
engram.structured_data = {
...engram.structured_data ?? {},
_demoted: { from, to: "local", patterns }
};
}
async function executeDedupDecision(deps, statement, context, decision, targetId) {
switch (decision) {
case "NOOP": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing) return { engram: existing, decision: "NOOP", existing_id: targetId };
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "UPDATE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const updated = { ...engrams[idx] };
updated.statement = statement;
updated.content_hash = computeContentHash(statement);
updated.engram_version = (updated.engram_version ?? 1) + 1;
updated.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) updated.tags = [.../* @__PURE__ */ new Set([...updated.tags, ...context.tags])];
demoteIfSensitive(deps, updated, updated.statement);
engrams[idx] = updated;
await persistOne(deps, engrams, updated);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_updated",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { old_statement: existing.statement, new_statement: statement, reason: "LLM dedup UPDATE" }
});
return { engram: updated, decision: "UPDATE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "MERGE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const merged = { ...engrams[idx] };
merged.statement = `${merged.statement} ${statement}`;
merged.content_hash = computeContentHash(merged.statement);
merged.engram_version = (merged.engram_version ?? 1) + 1;
merged.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) merged.tags = [.../* @__PURE__ */ new Set([...merged.tags, ...context.tags])];
if (0.7 > merged.activation.retrieval_strength) merged.activation.retrieval_strength = 0.7;
demoteIfSensitive(deps, merged, merged.statement);
engrams[idx] = merged;
await persistOne(deps, engrams, merged);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_merged",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { merged_statement: statement, reason: "LLM dedup MERGE" }
});
return { engram: merged, decision: "MERGE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "ADD":
default:
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
}
async function learnAsync(deps, statement, context) {
const hashMatch = await deps.hashDedup(statement, context?.scope);
if (hashMatch) {
return { engram: hashMatch, decision: "NOOP", existing_id: hashMatch.id };
}
const { enabled = true, threshold = 0.85, mode = "llm" } = deps.dedupConfig;
if (!enabled || mode === "off") {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
let candidates = [];
try {
candidates = await deps.recallHybrid(statement, { limit: 5 });
} catch {
candidates = await deps.recall(statement, { limit: 5 });
}
if (candidates.length === 0) {
candidates = await deps.recall(statement, { limit: 5 });
}
candidates = candidates.filter((c) => c.status === "active");
if (context?.scope) {
candidates = candidates.filter((c) => c.scope === context.scope);
}
if (candidates.length === 0) {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
const llm = context?.llm;
let decision = "ADD";
let targetId = null;
if (mode === "llm" && llm && deps.isLlmAvailable()) {
try {
const prompt = buildDedupPrompt(
statement,
candidates.map((c) => ({ id: c.id, statement: c.statement, type: c.type, domain: c.domain }))
);
const response = await llm(prompt);
const parsed = parseDedupResponse(response);
decision = parsed.decision;
targetId = parsed.target_id;
deps.recordLlmSuccess();
} catch (err) {
logger.warning(`LLM dedup failed, falling back to cosine: ${err}`);
deps.recordLlmFailure();
decision = "ADD";
}
}
return executeDedupDecision(deps, statement, context, decision, targetId);
}
async function learnBatch(deps, statements, llm, opts = {}) {
const results = [];
const failures = [];
const stats = { added: 0, updated: 0, merged: 0, noops: 0, failed: 0 };
const maxLlmCalls = opts.maxLlmCalls ?? 50;
let llmCallsUsed = 0;
let capWarned = false;
for (let i = 0; i < statements.length; i++) {
const { statement, context } = statements[i];
const stmtLlm = context?.llm ?? llm;
let effectiveLlm = stmtLlm;
if (stmtLlm) {
if (llmCallsUsed >= maxLlmCalls) {
effectiveLlm = void 0;
if (!capWarned) {
logger.warning(`learnBatch: maxLlmCalls (${maxLlmCalls}) reached \u2014 remaining statements use cosine/ADD dedup`);
capWarned = true;
}
} else {
effectiveLlm = async (prompt) => {
llmCallsUsed++;
return stmtLlm(prompt);
};
}
}
const ctx = { ...context, llm: effectiveLlm };
try {
const result = await learnAsync(deps, statement, ctx);
results.push({ ...result, input_index: i });
const key = result.decision.toLowerCase();
if (key === "noop") stats.noops++;
else if (key === "update") stats.updated++;
else if (key === "merge") stats.merged++;
else stats.added++;
} catch (err) {
stats.failed++;
failures.push({ index: i, statement, error: err instanceof Error ? err.message : String(err) });
logger.warning(`learnBatch: statement ${i} failed \u2014 ${err instanceof Error ? err.message : String(err)}`);
}
}
return { results, stats, failures };
}
export {
learnAsync,
learnBatch
};
+1
-1
{
"name": "@plur-ai/core",
"version": "0.16.1",
"version": "0.17.0",
"type": "module",

@@ -5,0 +5,0 @@ "main": "dist/index.js",

// src/fts.ts
var STOP_WORDS = /* @__PURE__ */ new Set([
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"are",
"was",
"were",
"been",
"have",
"has",
"not",
"but",
"its",
"you",
"your",
"can",
"will",
"should",
"would",
"could",
"may",
"might"
]);
function ftsTokenize(text) {
return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
}
function engramSearchText(engram) {
const parts = [engram.statement];
if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
if (engram.entities) {
for (const e of engram.entities) {
parts.push(e.name);
if (e.type !== "other") parts.push(e.type);
}
}
if (engram.temporal) {
if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
}
if (engram.rationale) parts.push(engram.rationale);
if (engram.source) parts.push(engram.source);
if (engram.dual_coding) {
if (engram.dual_coding.example) parts.push(engram.dual_coding.example);
if (engram.dual_coding.analogy) parts.push(engram.dual_coding.analogy);
}
if (engram.knowledge_anchors && engram.knowledge_anchors.length > 0) {
for (const a of engram.knowledge_anchors) {
if (a.snippet) parts.push(a.snippet);
}
}
return parts.join(" ");
}
function termMatches(t, qt) {
return t.includes(qt) || qt.startsWith(t);
}
function computeIdf(engrams, queryTokens, stats) {
if (stats) {
if (stats.N === 0) return /* @__PURE__ */ new Map();
const idf2 = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
const df = stats.df.get(qt) ?? 0;
idf2.set(qt, Math.max(0, Math.log(stats.N / (1 + df))));
}
return idf2;
}
const N = engrams.length;
if (N === 0) return /* @__PURE__ */ new Map();
const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
const idf = /* @__PURE__ */ new Map();
for (const qt of queryTokens) {
let df = 0;
for (const termSet of engramTermSets) {
if (termSet.has(qt) || Array.from(termSet).some((t) => termMatches(t, qt))) {
df++;
}
}
idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
}
return idf;
}
function extendCorpusStats(stats, queryTokens, outsiders) {
if (outsiders.length === 0) return stats;
const termSets = [];
let totalLen = 0;
for (const e of outsiders) {
const terms = ftsTokenize(engramSearchText(e));
totalLen += terms.length;
termSets.push(new Set(terms));
}
const df = new Map(stats.df);
for (const qt of queryTokens) {
let added = 0;
for (const set of termSets) {
if (set.has(qt) || Array.from(set).some((t) => termMatches(t, qt))) added++;
}
if (added > 0) df.set(qt, (df.get(qt) ?? 0) + added);
}
const N = stats.N + outsiders.length;
return {
N,
df,
avgDocLength: N > 0 ? (stats.avgDocLength * stats.N + totalLen) / N : 0
};
}
var BM25_K1 = 1.2;
var BM25_B = 0.75;
function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
const allTerms = ftsTokenize(engramSearchText(engram));
if (queryTokens.length === 0) return 0;
const docLen = allTerms.length;
const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
let score = 0;
for (const qt of queryTokens) {
let effectiveIdf;
if (!idfWeights) {
effectiveIdf = 1;
} else if (hasNonZeroIdf) {
effectiveIdf = idfWeights.get(qt) ?? 0;
if (effectiveIdf === 0) continue;
} else {
effectiveIdf = 1;
}
let tf = 0;
for (const t of allTerms) {
if (termMatches(t, qt)) tf++;
}
if (tf === 0) continue;
const numerator = tf * (BM25_K1 + 1);
const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
score += effectiveIdf * (numerator / denominator);
}
return score;
}
function searchEngrams(engrams, query, limit = 20, stats) {
const queryTokens = ftsTokenize(query);
if (queryTokens.length === 0) return [];
const idfWeights = computeIdf(engrams, queryTokens, stats);
const avgDocLength = stats ? stats.avgDocLength : engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
let scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0);
if (scored.length === 0) {
scored = engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, void 0, avgDocLength) })).filter((r) => r.score > 0);
}
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
}
export {
ftsTokenize,
engramSearchText,
termMatches,
computeIdf,
extendCorpusStats,
ftsScore,
searchEngrams
};
// src/async-mutex.ts
var AsyncMutex = class {
queue = Promise.resolve();
/** Number of runs queued or executing. Drives KeyedAsyncMutex eviction. */
depth = 0;
/** True when nothing is queued or running. */
get idle() {
return this.depth === 0;
}
async run(fn) {
let release;
const wait = new Promise((res) => {
release = res;
});
const prev = this.queue;
this.queue = prev.then(() => wait);
this.depth++;
await prev;
try {
return await fn();
} finally {
this.depth--;
release();
}
}
};
var KeyedAsyncMutex = class {
mutexes = /* @__PURE__ */ new Map();
/** Number of keys with work queued or running. Test/diagnostic seam. */
get size() {
return this.mutexes.size;
}
async run(key, fn) {
let mutex = this.mutexes.get(key);
if (!mutex) {
mutex = new AsyncMutex();
this.mutexes.set(key, mutex);
}
try {
return await mutex.run(fn);
} finally {
if (mutex.idle && this.mutexes.get(key) === mutex) this.mutexes.delete(key);
}
}
};
// src/history.ts
import * as fs from "fs";
import { join } from "path";
import { createHash } from "crypto";
function appendHistory(root, event) {
const historyDir = join(root, "history");
if (!fs.existsSync(historyDir)) {
fs.mkdirSync(historyDir, { recursive: true });
}
const date = event.timestamp.slice(0, 7);
const filePath = join(historyDir, `${date}.jsonl`);
const line = JSON.stringify(event) + "\n";
fs.appendFileSync(filePath, line, "utf8");
}
function readHistory(root, yearMonth) {
const filePath = join(root, "history", `${yearMonth}.jsonl`);
if (!fs.existsSync(filePath)) return [];
const content = fs.readFileSync(filePath, "utf8");
const lines = content.split("\n").filter((l) => l.trim().length > 0);
const events = [];
for (const line of lines) {
try {
events.push(JSON.parse(line));
} catch {
}
}
return events;
}
function listHistoryMonths(root) {
const historyDir = join(root, "history");
if (!fs.existsSync(historyDir)) return [];
return fs.readdirSync(historyDir).filter((f) => f.endsWith(".jsonl")).map((f) => f.replace(".jsonl", "")).sort();
}
function readHistoryForEngram(root, engramId) {
const months = listHistoryMonths(root);
const events = [];
for (const month of months) {
const monthEvents = readHistory(root, month);
for (const event of monthEvents) {
if (event.engram_id === engramId) {
events.push(event);
}
}
}
return events;
}
var _PROC_SALT = (process.pid % 1296).toString(36).padStart(2, "0");
var _evtSeq = 0;
var _injSeq = 0;
function generateEventId() {
return `EVT-${Date.now()}-${_PROC_SALT}${(_evtSeq++).toString(36).padStart(4, "0")}`;
}
function generateInjectionId() {
return `INJ-${Date.now()}-${_PROC_SALT}${(_injSeq++).toString(36).padStart(4, "0")}`;
}
function computeQueryHash(task) {
const normalized = task.toLowerCase().replace(/\s+/g, " ").trim();
return createHash("sha256").update(normalized).digest("hex").slice(0, 16);
}
function findLatestInjectionFor(root, engramId, maxMonths = 2) {
const now = /* @__PURE__ */ new Date();
const allowed = /* @__PURE__ */ new Set();
for (let i = 0; i < maxMonths; i++) {
const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1));
allowed.add(d.toISOString().slice(0, 7));
}
const months = listHistoryMonths(root).filter((m) => allowed.has(m)).reverse();
for (const month of months) {
let latest = null;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const ids = event.data.ids;
if (!Array.isArray(ids) || !ids.includes(engramId)) continue;
if (!latest || event.timestamp > latest.timestamp) latest = event;
}
if (latest) return { injection_id: latest.engram_id, timestamp: latest.timestamp };
}
return null;
}
var INJECTION_SOURCES = /* @__PURE__ */ new Set([
"session_start",
"inject",
"hook",
"unknown"
]);
var ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
function readCoInjections(root, months) {
const events = [];
let skipped = 0;
const wanted = months ? new Set(months) : null;
for (const month of listHistoryMonths(root)) {
if (wanted && !wanted.has(month)) continue;
for (const event of readHistory(root, month)) {
if (event.event !== "co_injection") continue;
const raw = event.data;
if (!Array.isArray(raw.ids) || typeof raw.query_hash !== "string") {
skipped++;
continue;
}
if (typeof event.timestamp !== "string" || !ISO_TIMESTAMP.test(event.timestamp)) {
skipped++;
continue;
}
const ids = raw.ids.filter((id) => typeof id === "string" && id.length > 0);
if (ids.length !== raw.ids.length) skipped++;
const data = { ids, query_hash: raw.query_hash };
if (typeof raw.tokens_used === "number" && Number.isFinite(raw.tokens_used)) {
data.tokens_used = raw.tokens_used;
}
if (raw.source !== void 0) {
data.source = INJECTION_SOURCES.has(raw.source) ? raw.source : "unknown";
}
if (typeof raw.scope === "string") data.scope = raw.scope;
if (typeof raw.session_id === "string") data.session_id = raw.session_id;
events.push({ injection_id: event.engram_id, timestamp: event.timestamp, data });
}
}
events.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
return { events, skipped };
}
function countInjectionEvents(root) {
const counts = {
co_injection: 0,
injection_outcome: 0,
outcome_positive: 0,
outcome_negative: 0
};
for (const month of listHistoryMonths(root)) {
for (const event of readHistory(root, month)) {
if (event.event === "co_injection") {
counts.co_injection++;
} else if (event.event === "injection_outcome") {
counts.injection_outcome++;
if (event.data.signal === "positive") counts.outcome_positive++;
else if (event.data.signal === "negative") counts.outcome_negative++;
}
}
}
return counts;
}
// src/content-hash.ts
import { createHash as createHash2 } from "crypto";
function normalizeStatement(statement) {
return statement.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
}
function computeContentHash(statement) {
const normalized = normalizeStatement(statement);
return createHash2("sha256").update(normalized).digest("hex");
}
// src/store/async-lock.ts
import { writeFile, unlink, stat } from "fs/promises";
import { constants } from "fs";
import * as path from "path";
var processLocks = new KeyedAsyncMutex();
function sleep(ms) {
return new Promise((res) => setTimeout(res, ms));
}
async function withFileLock(filePath, fn, options) {
const lockPath = filePath + ".lock";
const maxRetries = options?.maxRetries ?? 5;
const baseDelay = options?.baseDelay ?? 100;
const staleThreshold = options?.staleThreshold ?? 1e4;
let acquired = false;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
await writeFile(lockPath, `${process.pid}`, { flag: constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL });
acquired = true;
break;
} catch (err) {
if (err.code !== "EEXIST") throw err;
try {
const s = await stat(lockPath);
if (Date.now() - s.mtimeMs > staleThreshold) {
await unlink(lockPath).catch(() => {
});
continue;
}
} catch {
continue;
}
if (attempt === maxRetries) {
throw new Error(`Failed to acquire lock on ${filePath} after ${maxRetries} retries`);
}
const delay = Math.min(baseDelay * Math.pow(2, attempt), 5e3);
await sleep(delay);
}
}
if (!acquired) {
throw new Error(
`Failed to acquire lock on ${filePath} after ${maxRetries} retries (contended throughout)`
);
}
try {
return await fn();
} finally {
await unlink(lockPath).catch(() => {
});
}
}
async function withAsyncLock(filePath, fn, options) {
return processLocks.run(path.resolve(filePath), () => withFileLock(filePath, fn, options));
}
// src/dedup.ts
function buildDedupPrompt(newStatement, candidates) {
const candidateList = candidates.map(
(c, i) => `${i + 1}. [${c.id}] (${c.type}${c.domain ? ", domain: " + c.domain : ""})
"${c.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare a new memory statement against existing ones.
NEW STATEMENT:
"${newStatement}"
EXISTING ENGRAMS:
${candidateList}
For each existing engram, answer:
1. RELATIONSHIP: Is the new statement a DUPLICATE (same meaning), EVOLUTION (updated version of same knowledge), COMPLEMENTARY (related but different angle), or UNRELATED?
2. RICHNESS: Does the new statement contain more specific, actionable information than the existing one? (yes/no)
Then give your OVERALL DECISION (exactly one):
- NOOP: New statement is an exact duplicate of an existing engram. Return the ID.
- UPDATE: New statement is an evolution with MORE information. Return the ID to update.
- MERGE: New statement and an existing one are complementary \u2014 combining them preserves both. Return the ID to merge with.
- ADD: New statement is genuinely new knowledge.
Respond in this exact format:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID if UPDATE/MERGE/NOOP, or "none" if ADD>
REASON: <one sentence explanation>`;
}
function buildBatchDedupPrompt(statements, existingEngrams) {
const stmtList = statements.map((s, i) => `${i + 1}. "${s}"`).join("\n");
const engramList = existingEngrams.map(
(e, i) => `${i + 1}. [${e.id}] (${e.type}${e.domain ? ", domain: " + e.domain : ""})
"${e.statement}"`
).join("\n");
return `You are a memory deduplication system. Compare NEW statements against existing engrams.
NEW STATEMENTS:
${stmtList}
EXISTING ENGRAMS:
${engramList}
For each NEW statement, decide:
- NOOP: Exact duplicate of an existing engram.
- UPDATE: Evolution with more info than existing.
- MERGE: Complementary with existing \u2014 combine.
- ADD: Genuinely new knowledge.
Respond with one block per new statement:
STATEMENT_1:
DECISION: <ADD|UPDATE|MERGE|NOOP>
TARGET: <engram ID or "none">
STATEMENT_2:
...`;
}
function parseDedupResponse(response) {
const decisionMatch = response.match(/DECISION:\s*(ADD|UPDATE|MERGE|NOOP)/i);
const targetMatch = response.match(/TARGET:\s*([^\n]+)/i);
const reasonMatch = response.match(/REASON:\s*([^\n]+)/i);
const decision = decisionMatch?.[1]?.toUpperCase() ?? "ADD";
const targetRaw = targetMatch?.[1]?.trim() ?? "none";
const target_id = targetRaw === "none" ? null : targetRaw.replace(/[^A-Za-z0-9-]/g, "");
const reason = reasonMatch?.[1]?.trim() ?? "";
return { decision, target_id, reason };
}
export {
AsyncMutex,
KeyedAsyncMutex,
appendHistory,
readHistory,
listHistoryMonths,
readHistoryForEngram,
generateEventId,
generateInjectionId,
computeQueryHash,
findLatestInjectionFor,
readCoInjections,
countInjectionEvents,
normalizeStatement,
computeContentHash,
withAsyncLock,
buildDedupPrompt,
buildBatchDedupPrompt,
parseDedupResponse
};
import {
engramSearchText
} from "./chunk-JDQYKS6P.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/embeddings.ts
import { existsSync as existsSync2, readFileSync as readFileSync2, mkdirSync as mkdirSync2 } from "fs";
import { join as join2, dirname as dirname2 } from "path";
import { createHash } from "crypto";
// src/sync.ts
import { execFileSync } from "child_process";
import { existsSync, readFileSync, writeFileSync, renameSync, mkdirSync, unlinkSync, statSync, readdirSync } from "fs";
import { join, dirname, relative } from "path";
import * as yaml from "js-yaml";
// src/scope-util.ts
var SHARED_SCOPE_PREFIXES = ["group:", "project:", "space:", "team:", "org:", "public"];
function isSharedScope(scope) {
const s = scope.toLowerCase();
return SHARED_SCOPE_PREFIXES.some(
(p) => p === "public" ? s === "public" || s.startsWith("public:") || s.startsWith("public/") : s.startsWith(p)
);
}
function isPersonalScope(scope) {
return !isSharedScope(scope);
}
function isScopeWithin(scope, queryScope) {
return scope === queryScope || scope.startsWith(queryScope + ":") || scope.startsWith(queryScope + "/");
}
function scopeAllowFilter(scopes) {
if (scopes === void 0) return () => true;
const allowed = new Set(scopes);
return (scope) => allowed.has(scope);
}
// src/sync.ts
var GITIGNORE = `# PLUR \u2014 secrets (machine-local, NEVER synced)
config.yaml
secrets.yaml
agent-keystore.json
*.token
# PLUR \u2014 derived/cache files (regenerated automatically)
embeddings/
.embeddings-cache.json
*.db
*.sqlite
store.pglite/
exchange/
`;
var SYNC_PATHS = ["engrams.yaml", "episodes.yaml", "candidates.yaml", "tensions.yaml", "packs", ".gitignore"];
var SECRET_PATHS = ["config.yaml", "secrets.yaml", "agent-keystore.json"];
var PACK_ALLOW_NAMES = ["SKILL.md", "engrams.yaml", "INTEGRITY", "metadata.json"];
function git(args, cwd) {
return execFileSync("git", args, { cwd, encoding: "utf8", timeout: 3e4 }).trim();
}
function gitSafe(args, cwd) {
try {
return git(args, cwd);
} catch {
return null;
}
}
function isGitRepo(root) {
return existsSync(join(root, ".git"));
}
function hasGitCli() {
try {
execFileSync("git", ["--version"], { encoding: "utf8", timeout: 5e3 });
return true;
} catch {
return false;
}
}
function getRemote(root) {
return gitSafe(["remote", "get-url", "origin"], root);
}
function isDirty(root) {
const status = gitSafe(["status", "--porcelain"], root);
return status !== null && status.length > 0;
}
function countDiff(root, direction) {
const tracking = gitSafe(["rev-parse", "--abbrev-ref", "@{u}"], root);
if (!tracking) return 0;
const flag = direction === "ahead" ? "--left-only" : "--right-only";
const count = gitSafe(["rev-list", flag, "--count", "HEAD...@{u}"], root);
return count ? parseInt(count, 10) : 0;
}
function getSyncStatus(root) {
if (!isGitRepo(root)) {
return { initialized: false, remote: null, dirty: false, branch: null, ahead: 0, behind: 0 };
}
const branch = gitSafe(["rev-parse", "--abbrev-ref", "HEAD"], root);
const remote = getRemote(root);
if (remote) gitSafe(["fetch", "origin", "--quiet"], root);
return {
initialized: true,
remote,
dirty: isDirty(root),
branch,
ahead: countDiff(root, "ahead"),
behind: countDiff(root, "behind")
};
}
function stageStoreFiles(root) {
for (const secret of SECRET_PATHS) {
gitSafe(["rm", "--cached", "--ignore-unmatch", "--quiet", "--", secret], root);
}
const present = SYNC_PATHS.filter((p) => p !== "packs" && existsSync(join(root, p)));
const pathspecs = [...present, ...packStorePaths(root)];
if (pathspecs.length > 0) {
git(["add", "-A", "-f", "--", ...pathspecs], root);
}
const staged = gitSafe(["diff", "--cached", "--name-only"], root);
return staged ? staged.split("\n").filter(Boolean).length : 0;
}
function packStorePaths(root) {
const packsDir = join(root, "packs");
if (!existsSync(packsDir)) return [];
const allow = new Set(PACK_ALLOW_NAMES);
const paths = /* @__PURE__ */ new Set();
const stack = [packsDir];
while (stack.length > 0) {
const dir = stack.pop();
for (const ent of readdirSync(dir, { withFileTypes: true })) {
const full = join(dir, ent.name);
if (ent.isDirectory()) stack.push(full);
else if (allow.has(ent.name)) paths.add(relative(root, full));
}
}
const tracked = gitSafe(["ls-files", "--", "packs"], root);
if (tracked) {
for (const f of tracked.split("\n").filter(Boolean)) {
if (allow.has(f.split("/").pop() ?? "")) paths.add(f);
}
}
return [...paths];
}
var YAML_DUMP_OPTS = { lineWidth: 120, noRefs: true, quotingType: '"' };
function readEngramList(root) {
const path = join(root, "engrams.yaml");
if (!existsSync(path)) return null;
let raw;
try {
raw = yaml.load(readFileSync(path, "utf8"));
} catch {
return null;
}
if (Array.isArray(raw)) return { raw, list: raw };
if (raw && typeof raw === "object" && Array.isArray(raw.engrams)) {
return { raw, list: raw.engrams };
}
return null;
}
function pushKeep(remoteType) {
if (remoteType === "shared") {
return (e) => isSharedScope(e?.scope ?? "") && (e?.visibility ?? "private") !== "private";
}
return (e) => e?.scope !== "local";
}
function stripWarning(root, remoteType) {
const parsed = readEngramList(root);
if (!parsed) return void 0;
if (remoteType === "shared") {
const stripped = parsed.list.filter((e) => !pushKeep("shared")(e)).length;
if (stripped === 0) return void 0;
return `Shared remote: pushed only shared-scope, non-private engrams \u2014 ${stripped} personal-scope or private-visibility engram(s) stayed local.`;
}
const count = parsed.list.filter(
(e) => e?.scope !== "local" && (e?.visibility ?? "private") === "private"
).length;
if (count === 0) return void 0;
return `Note: sync remote receives all engrams including ${count} private-visibility one(s) \u2014 use a private git remote. For a team remote, set sync.remote_type: shared to exclude them.`;
}
function stageStripped(root, remoteType) {
const parsed = readEngramList(root);
if (!parsed) return;
const { raw, list } = parsed;
const keep = pushKeep(remoteType);
const filtered = list.filter(keep);
if (filtered.length === list.length) return;
const out = Array.isArray(raw) ? yaml.dump(filtered, YAML_DUMP_OPTS) : yaml.dump({ ...raw, engrams: filtered }, YAML_DUMP_OPTS);
const hash = execFileSync("git", ["hash-object", "-w", "--stdin"], {
cwd: root,
input: out,
encoding: "utf8",
timeout: 3e4
}).trim();
git(["update-index", "--cacheinfo", `100644,${hash},engrams.yaml`], root);
}
function initRepo(root, remoteType) {
git(["init"], root);
atomicWrite(join(root, ".gitignore"), GITIGNORE);
stageStoreFiles(root);
stageStripped(root, remoteType);
git(["commit", "-m", "Initial PLUR engram store"], root);
}
function commitChanges(root, remoteType) {
const filesChanged = stageStoreFiles(root);
if (filesChanged === 0) return 0;
stageStripped(root, remoteType);
const diff = gitSafe(["diff", "--cached", "--shortstat"], root);
if (!diff || diff.length === 0) return 0;
const now = (/* @__PURE__ */ new Date()).toISOString().slice(0, 19).replace("T", " ");
git(["commit", "-m", `plur sync ${now}`], root);
const match = diff.match(/(\d+) file/);
return match ? parseInt(match[1], 10) : filesChanged;
}
function hasConflictMarkers(root) {
const result = gitSafe(["grep", "-l", "<<<<<<<"], root);
return result !== null && result.length > 0;
}
function pullRebase(root, remoteType) {
const branch = gitSafe(["rev-parse", "--abbrev-ref", "HEAD"], root) || "main";
const result = gitSafe(["pull", "--rebase", "origin", branch], root);
if (result !== null) return true;
gitSafe(["rebase", "--abort"], root);
const mergeResult = gitSafe(["pull", "origin", branch, "--no-edit"], root);
if (mergeResult !== null) return true;
if (hasConflictMarkers(root)) {
gitSafe(["merge", "--abort"], root);
throw new Error("Sync conflict: YAML files have merge conflicts that require manual resolution. Your local changes are preserved.");
}
stageStoreFiles(root);
stageStripped(root, remoteType);
gitSafe(["commit", "-m", "plur sync: merge conflict resolved (kept both)"], root);
return true;
}
function sync(root, remote, options) {
if (!hasGitCli()) {
throw new Error("git is not installed. Install git to enable sync.");
}
const remoteType = options?.remoteType ?? "personal";
if (!isGitRepo(root)) {
initRepo(root, remoteType);
if (remote) {
git(["remote", "add", "origin", remote], root);
const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], root);
git(["push", "-u", "origin", branch], root);
return { action: "initialized", message: `Initialized and pushed to ${remote}`, remote, files_changed: 0, warning: stripWarning(root, remoteType) };
}
return {
action: "initialized",
message: "Initialized local git repo. Call plur.sync with remote to enable cross-device sync.",
remote: null,
files_changed: 0
};
}
const existingRemote = getRemote(root);
if (remote && !existingRemote) {
git(["remote", "add", "origin", remote], root);
const filesChanged2 = commitChanges(root, remoteType);
const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], root);
git(["push", "-u", "origin", branch], root);
return { action: "synced", message: `Remote added and pushed to ${remote}`, remote, files_changed: filesChanged2, warning: stripWarning(root, remoteType) };
}
if (!existingRemote) {
const filesChanged2 = commitChanges(root, remoteType);
if (filesChanged2 === 0) {
return { action: "up-to-date", message: 'No changes to commit. Add a remote with await plur.sync({ remote: "..." }) to enable cross-device sync.', remote: null, files_changed: 0 };
}
return { action: "committed", message: `Committed ${filesChanged2} file(s) locally.`, remote: null, files_changed: filesChanged2 };
}
const filesChanged = commitChanges(root, remoteType);
gitSafe(["fetch", "origin", "--quiet"], root);
const behind = countDiff(root, "behind");
const aheadBefore = countDiff(root, "ahead");
if (behind > 0) {
pullRebase(root, remoteType);
}
let pushError = null;
const aheadAfter = countDiff(root, "ahead");
if (aheadAfter > 0) {
try {
git(["push", "origin"], root);
} catch (err) {
pushError = (err.message || "").trim() || "git push failed";
}
}
if (filesChanged === 0 && behind === 0 && aheadBefore === 0) {
return { action: "up-to-date", message: "Already in sync.", remote: existingRemote, files_changed: 0, warning: stripWarning(root, remoteType) };
}
const parts = [];
if (filesChanged > 0) parts.push(`${filesChanged} file(s) committed`);
if (behind > 0) parts.push(`pulled ${behind} remote commit(s)`);
if (aheadAfter === 0 && aheadBefore > 0) parts.push("pushed");
if (pushError) parts.push("NOT pushed \u2014 the commit is local only");
return {
action: "synced",
message: `Synced. ${parts.join(", ")}.`,
remote: existingRemote,
files_changed: filesChanged,
warning: stripWarning(root, remoteType),
...pushError ? { push_error: pushError } : {}
};
}
function withLock(filePath, fn, options) {
const lockPath = filePath + ".lock";
const maxRetries = options?.maxRetries ?? 5;
const baseDelay = options?.baseDelay ?? 100;
const staleThreshold = options?.staleThreshold ?? 1e4;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
writeFileSync(lockPath, `${process.pid}`, { flag: "wx" });
break;
} catch (err) {
if (err.code !== "EEXIST") throw err;
try {
const stat = statSync(lockPath);
if (Date.now() - stat.mtimeMs > staleThreshold) {
unlinkSync(lockPath);
continue;
}
} catch {
continue;
}
if (attempt === maxRetries) {
throw new Error(`Failed to acquire lock on ${filePath} after ${maxRetries} retries`);
}
const delay = baseDelay * Math.pow(2, attempt);
const end = Date.now() + delay;
while (Date.now() < end) {
}
}
}
try {
return fn();
} finally {
try {
unlinkSync(lockPath);
} catch {
}
}
}
function atomicWrite(filePath, content) {
const dir = dirname(filePath);
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
const tmp = filePath + ".tmp";
writeFileSync(tmp, content);
renameSync(tmp, filePath);
}
// src/embeddings.ts
var EMBED_DIM = 384;
var embedPipeline = null;
var lastLoadError = null;
var transformersUnavailable = false;
function readDisabledFromEnv(env) {
const raw = env.PLUR_DISABLE_EMBEDDINGS;
if (!raw) return null;
const normalized = raw.trim().toLowerCase();
if (normalized === "1" || normalized === "true" || normalized === "yes") {
return "embeddings disabled by PLUR_DISABLE_EMBEDDINGS env var";
}
return null;
}
var ENV_DISABLED_REASON = readDisabledFromEnv(process.env);
var embeddingsDisabled = ENV_DISABLED_REASON !== null;
var disabledReason = ENV_DISABLED_REASON;
function embedderStatus() {
return {
available: !embeddingsDisabled && !transformersUnavailable,
loaded: embedPipeline !== null,
lastError: lastLoadError,
disabled: embeddingsDisabled,
disabledReason
};
}
function setEmbeddingsEnabled(enabled, reason) {
embeddingsDisabled = !enabled;
disabledReason = enabled ? null : reason ?? "embeddings disabled by config";
if (!enabled) {
embedPipeline = null;
}
}
function resetEmbedder() {
transformersUnavailable = false;
lastLoadError = null;
embedPipeline = null;
}
function _setCachedEmbedder(adapter) {
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
}
async function getEmbedder() {
if (embeddingsDisabled) return null;
if (embedPipeline) return embedPipeline;
try {
const { getEmbedder: getAdapter, resolveEmbedderName } = await import("./embedders-TB252LRE.js");
const adapter = getAdapter(resolveEmbedderName());
embedPipeline = adapter;
transformersUnavailable = false;
lastLoadError = null;
return embedPipeline;
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
return null;
}
}
async function embed(text, role) {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.embed === "function") {
let vector;
try {
vector = await embedder.embed(text, role);
} catch (err) {
transformersUnavailable = true;
lastLoadError = err instanceof Error ? err.message : String(err);
embedPipeline = null;
return null;
}
if (vector && typeof embedder.dim === "number" && vector.length !== embedder.dim) {
throw new Error(
`Embedding dimension mismatch: embedder "${embedder.name}" declares ${embedder.dim} dims but produced ${vector.length}. The adapter's declared dim and its model must agree; vectors at the wrong dimension are incompatible with any store that persisted them.`
);
}
return vector;
}
const result = await embedder(text, { pooling: "cls", normalize: true });
return new Float32Array(result.data);
}
async function getActiveEmbedderMeta() {
const embedder = await getEmbedder();
if (!embedder) return null;
if (typeof embedder.name === "string" && typeof embedder.dim === "number") {
return { name: embedder.name, dim: embedder.dim };
}
return { name: "legacy-pipeline", dim: 0 };
}
async function activeEmbedderDim() {
const meta = await getActiveEmbedderMeta();
return meta && meta.dim > 0 ? meta.dim : null;
}
function cosineSimilarity(a, b) {
let dot = 0;
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
return dot;
}
var CACHE_VERSION = 1;
function emptyCache(meta) {
return {
meta: {
embedder_name: meta.name,
embedder_dim: meta.dim,
version: CACHE_VERSION
},
entries: {}
};
}
function loadCache(cachePath, active) {
if (!existsSync2(cachePath)) return emptyCache(active);
try {
const raw = JSON.parse(readFileSync2(cachePath, "utf8"));
if (!raw || typeof raw !== "object" || !raw.meta) {
logger.info(`[embeddings] cache at ${cachePath} is in legacy format (no embedder meta) \u2014 rebuilding for active embedder ${active.name} (${active.dim}d).`);
return emptyCache(active);
}
const meta = raw.meta;
if (meta.embedder_name !== active.name || meta.embedder_dim !== active.dim) {
logger.info(`[embeddings] cache embedder mismatch \u2014 on-disk: ${meta.embedder_name} (${meta.embedder_dim}d), active: ${active.name} (${active.dim}d). Rebuilding cache.`);
return emptyCache(active);
}
const entries = raw.entries && typeof raw.entries === "object" ? raw.entries : {};
return { meta: { embedder_name: meta.embedder_name, embedder_dim: meta.embedder_dim, version: meta.version ?? CACHE_VERSION }, entries };
} catch {
return emptyCache(active);
}
}
function saveCache(cachePath, cache) {
const dir = dirname2(cachePath);
if (dir && !existsSync2(dir)) mkdirSync2(dir, { recursive: true });
atomicWrite(cachePath, JSON.stringify(cache));
}
function hashStatement(statement) {
return createHash("sha256").update(statement).digest("hex").slice(0, 16);
}
async function embeddingSearch(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join2(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const score = cosineSimilarity(queryEmbedding, engramEmbedding);
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit).map((s) => s.engram);
}
async function embeddingSearchWithScores(engrams, query, limit, storagePath) {
if (engrams.length === 0) return [];
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) return [];
const cachePath = storagePath ? join2(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
const cache = loadCache(cachePath, activeMeta);
const queryEmbedding = await embed(query, "query");
if (!queryEmbedding) {
return [];
}
const similarities = [];
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
let engramEmbedding;
if (cache.entries[engram.id]?.hash === hash) {
engramEmbedding = new Float32Array(cache.entries[engram.id].embedding);
} else {
const emb = await embed(searchText);
if (!emb) return [];
engramEmbedding = emb;
cache.entries[engram.id] = {
hash,
embedding: Array.from(engramEmbedding)
};
}
const rawScore = cosineSimilarity(queryEmbedding, engramEmbedding);
const score = Math.max(0, Math.min(1, rawScore));
similarities.push({ engram, score });
}
saveCache(cachePath, cache);
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, limit);
}
async function rebuildJsonCache(engrams, storagePath, opts) {
const activeMeta = await getActiveEmbedderMeta();
if (!activeMeta) {
return { reembedded: 0, skipped: true, reason: "embedder unavailable" };
}
const cachePath = join2(storagePath, ".embeddings-cache.json");
const cache = opts?.full ? emptyCache(activeMeta) : loadCache(cachePath, activeMeta);
let count = 0;
for (const engram of engrams) {
const searchText = engramSearchText(engram);
const hash = hashStatement(searchText);
if (cache.entries[engram.id]?.hash === hash && !opts?.full) continue;
const vec = await embed(searchText);
if (!vec) {
return { reembedded: count, skipped: true, reason: "embedder unavailable mid-rebuild" };
}
cache.entries[engram.id] = { hash, embedding: Array.from(vec) };
count++;
}
saveCache(cachePath, cache);
return { reembedded: count, skipped: false };
}
export {
SHARED_SCOPE_PREFIXES,
isSharedScope,
isPersonalScope,
isScopeWithin,
scopeAllowFilter,
getSyncStatus,
sync,
withLock,
atomicWrite,
EMBED_DIM,
readDisabledFromEnv,
embedderStatus,
setEmbeddingsEnabled,
resetEmbedder,
_setCachedEmbedder,
embed,
activeEmbedderDim,
cosineSimilarity,
embeddingSearch,
embeddingSearchWithScores,
rebuildJsonCache
};
import {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
} from "./chunk-WSELEVKD.js";
import "./chunk-JDQYKS6P.js";
import "./chunk-E4YVUWMJ.js";
export {
EMBED_DIM,
_setCachedEmbedder,
activeEmbedderDim,
cosineSimilarity,
embed,
embedderStatus,
embeddingSearch,
embeddingSearchWithScores,
readDisabledFromEnv,
rebuildJsonCache,
resetEmbedder,
setEmbeddingsEnabled
};
import {
computeIdf,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
searchEngrams,
termMatches
} from "./chunk-JDQYKS6P.js";
export {
computeIdf,
engramSearchText,
extendCorpusStats,
ftsScore,
ftsTokenize,
searchEngrams,
termMatches
};
import {
appendHistory,
buildDedupPrompt,
computeContentHash,
parseDedupResponse,
withAsyncLock
} from "./chunk-OJGZ6OWP.js";
import {
logger
} from "./chunk-E4YVUWMJ.js";
// src/learn-async.ts
async function withStoreLock(deps, fn) {
if (deps.store.withExclusiveAccess) return await deps.store.withExclusiveAccess(fn);
return await withAsyncLock(deps.engramsPath, fn);
}
function demoteIfSensitive(deps, engram, newStatement) {
const tags = Array.isArray(engram.tags) ? engram.tags.filter((t) => typeof t === "string") : [];
const scanText = tags.length ? `${newStatement}
${tags.join(" ")}` : newStatement;
const offending = deps.offendingHitsForScope(scanText, engram.scope ?? "global");
if (offending.length === 0) return;
const patterns = [...new Set(offending.map((h) => h.pattern))].join(", ");
logger.warning(
`[plur] sensitive content (${patterns}) held back from shared scope "${engram.scope}" \u2014 demoted to local/private so it is not written to a shared store. Re-scope deliberately if this is a false positive.`
);
const from = engram.scope ?? "global";
engram.scope = "local";
engram.visibility = "private";
engram.structured_data = {
...engram.structured_data ?? {},
_demoted: { from, to: "local", patterns }
};
}
async function executeDedupDecision(deps, statement, context, decision, targetId) {
switch (decision) {
case "NOOP": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing) return { engram: existing, decision: "NOOP", existing_id: targetId };
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "UPDATE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const updated = { ...engrams[idx] };
updated.statement = statement;
updated.content_hash = computeContentHash(statement);
updated.engram_version = (updated.engram_version ?? 1) + 1;
updated.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) updated.tags = [.../* @__PURE__ */ new Set([...updated.tags, ...context.tags])];
demoteIfSensitive(deps, updated, updated.statement);
engrams[idx] = updated;
await deps.store.save(engrams);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_updated",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { old_statement: existing.statement, new_statement: statement, reason: "LLM dedup UPDATE" }
});
return { engram: updated, decision: "UPDATE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "MERGE": {
if (targetId) {
const existing = await deps.getById(targetId);
if (existing && existing.commitment !== "locked") {
const result = await withStoreLock(deps, async () => {
const engrams = await deps.store.load();
const idx = engrams.findIndex((e) => e.id === targetId);
if (idx === -1) return null;
const merged = { ...engrams[idx] };
merged.statement = `${merged.statement} ${statement}`;
merged.content_hash = computeContentHash(merged.statement);
merged.engram_version = (merged.engram_version ?? 1) + 1;
merged.activation.last_accessed = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
if (context?.tags) merged.tags = [.../* @__PURE__ */ new Set([...merged.tags, ...context.tags])];
if (0.7 > merged.activation.retrieval_strength) merged.activation.retrieval_strength = 0.7;
demoteIfSensitive(deps, merged, merged.statement);
engrams[idx] = merged;
await deps.store.save(engrams);
await deps.syncIndex();
appendHistory(deps.rootPath, {
event: "engram_merged",
engram_id: targetId,
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
data: { merged_statement: statement, reason: "LLM dedup MERGE" }
});
return { engram: merged, decision: "MERGE", existing_id: targetId };
});
if (result) return result;
}
}
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
case "ADD":
default:
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
}
async function learnAsync(deps, statement, context) {
const hashMatch = await deps.hashDedup(statement, context?.scope);
if (hashMatch) {
return { engram: hashMatch, decision: "NOOP", existing_id: hashMatch.id };
}
const { enabled = true, threshold = 0.85, mode = "llm" } = deps.dedupConfig;
if (!enabled || mode === "off") {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
let candidates = [];
try {
candidates = await deps.recallHybrid(statement, { limit: 5 });
} catch {
candidates = await deps.recall(statement, { limit: 5 });
}
if (candidates.length === 0) {
candidates = await deps.recall(statement, { limit: 5 });
}
candidates = candidates.filter((c) => c.status === "active");
if (context?.scope) {
candidates = candidates.filter((c) => c.scope === context.scope);
}
if (candidates.length === 0) {
return { engram: await deps.learn(statement, context), decision: "ADD" };
}
const llm = context?.llm;
let decision = "ADD";
let targetId = null;
if (mode === "llm" && llm && deps.isLlmAvailable()) {
try {
const prompt = buildDedupPrompt(
statement,
candidates.map((c) => ({ id: c.id, statement: c.statement, type: c.type, domain: c.domain }))
);
const response = await llm(prompt);
const parsed = parseDedupResponse(response);
decision = parsed.decision;
targetId = parsed.target_id;
deps.recordLlmSuccess();
} catch (err) {
logger.warning(`LLM dedup failed, falling back to cosine: ${err}`);
deps.recordLlmFailure();
decision = "ADD";
}
}
return executeDedupDecision(deps, statement, context, decision, targetId);
}
async function learnBatch(deps, statements, llm, opts = {}) {
const results = [];
const failures = [];
const stats = { added: 0, updated: 0, merged: 0, noops: 0, failed: 0 };
const maxLlmCalls = opts.maxLlmCalls ?? 50;
let llmCallsUsed = 0;
let capWarned = false;
for (let i = 0; i < statements.length; i++) {
const { statement, context } = statements[i];
const stmtLlm = context?.llm ?? llm;
let effectiveLlm = stmtLlm;
if (stmtLlm) {
if (llmCallsUsed >= maxLlmCalls) {
effectiveLlm = void 0;
if (!capWarned) {
logger.warning(`learnBatch: maxLlmCalls (${maxLlmCalls}) reached \u2014 remaining statements use cosine/ADD dedup`);
capWarned = true;
}
} else {
effectiveLlm = async (prompt) => {
llmCallsUsed++;
return stmtLlm(prompt);
};
}
}
const ctx = { ...context, llm: effectiveLlm };
try {
const result = await learnAsync(deps, statement, ctx);
results.push({ ...result, input_index: i });
const key = result.decision.toLowerCase();
if (key === "noop") stats.noops++;
else if (key === "update") stats.updated++;
else if (key === "merge") stats.merged++;
else stats.added++;
} catch (err) {
stats.failed++;
failures.push({ index: i, statement, error: err instanceof Error ? err.message : String(err) });
logger.warning(`learnBatch: statement ${i} failed \u2014 ${err instanceof Error ? err.message : String(err)}`);
}
}
return { results, stats, failures };
}
export {
learnAsync,
learnBatch
};

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display