🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

mdrip

Package Overview
Dependencies
Maintainers
1
Versions
7
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

mdrip - npm Package Compare versions

Comparing version
0.1.2
 to
0.1.3
+12
dist/api.d.ts
import { type FetchMarkdownOptions, type MarkdownResponse } from "./lib/cloudflare.js";
import type { FetchResult, PageEntry, SourcesIndex } from "./types.js";
export type { FetchMarkdownOptions, MarkdownResponse, FetchResult, PageEntry, SourcesIndex, };
export interface StoreFetchOptions extends FetchMarkdownOptions {
cwd?: string;
}
export declare function fetchMarkdown(url: string, options?: FetchMarkdownOptions): Promise<MarkdownResponse>;
export declare function fetchRawMarkdown(url: string, options?: FetchMarkdownOptions): Promise<string>;
export declare function fetchToStore(url: string, options?: StoreFetchOptions): Promise<FetchResult>;
export declare function fetchManyToStore(urls: string[], options?: StoreFetchOptions): Promise<FetchResult[]>;
export declare function listStoredPages(cwd?: string): Promise<PageEntry[]>;
//# sourceMappingURL=api.d.ts.map
+1
dist/api.d.ts.map
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAO7B,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvE,YAAY,EACV,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,SAAS,EACT,YAAY,GACb,CAAC;AAEF,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB;IAC7D,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA8ED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CAG3B;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,WAAW,CAAC,CAatB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,WAAW,EAAE,CAAC,CAiBxB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,SAAS,EAAE,CAAC,CAGtB"}
+109
dist/api.js
import { fetchMarkdownPage, } from "./lib/cloudflare.js";
import { listSources, savePageMarkdown, } from "./lib/storage.js";
import { updatePageIndex } from "./lib/agents.js";
import { normalizeUrl } from "./lib/url.js";
function mergeResults(existing, results) {
const now = new Date().toISOString();
const merged = [...existing];
for (const result of results) {
if (!result.success || !result.path) {
continue;
}
const entry = {
url: result.url,
resolvedUrl: result.resolvedUrl,
path: result.path,
fetchedAt: now,
status: result.status || 200,
contentType: result.contentType || "text/markdown",
markdownTokens: result.markdownTokens,
contentSignal: result.contentSignal,
source: result.source,
};
const index = merged.findIndex((page) => page.url === result.url);
if (index >= 0) {
merged[index] = entry;
}
else {
merged.push(entry);
}
}
return merged;
}
async function fetchOneForStore(spec, cwd, options) {
let normalizedUrl;
try {
normalizedUrl = normalizeUrl(spec);
}
catch (err) {
return {
url: spec,
path: "",
success: false,
error: err instanceof Error ? err.message : String(err),
};
}
try {
const response = await fetchMarkdownPage(normalizedUrl, options);
const storageUrl = normalizeUrl(response.resolvedUrl || normalizedUrl);
const outputPath = await savePageMarkdown(storageUrl, response.markdown, cwd);
return {
url: normalizedUrl,
resolvedUrl: response.resolvedUrl,
path: outputPath,
success: true,
status: response.status,
contentType: response.contentType,
markdownTokens: response.markdownTokens,
contentSignal: response.contentSignal,
source: response.source,
};
}
catch (err) {
return {
url: normalizedUrl,
path: "",
success: false,
error: err instanceof Error ? err.message : String(err),
};
}
}
export async function fetchMarkdown(url, options = {}) {
const normalizedUrl = normalizeUrl(url);
return fetchMarkdownPage(normalizedUrl, options);
}
export async function fetchRawMarkdown(url, options = {}) {
const response = await fetchMarkdown(url, options);
return response.markdown;
}
export async function fetchToStore(url, options = {}) {
const cwd = options.cwd || process.cwd();
const result = await fetchOneForStore(url, cwd, options);
if (!result.success) {
return result;
}
const existing = await listSources(cwd);
const pages = mergeResults(existing.pages, [result]);
await updatePageIndex({ pages }, cwd);
return result;
}
export async function fetchManyToStore(urls, options = {}) {
const cwd = options.cwd || process.cwd();
const results = [];
for (const spec of urls) {
const result = await fetchOneForStore(spec, cwd, options);
results.push(result);
}
const successful = results.filter((result) => result.success);
if (successful.length > 0) {
const existing = await listSources(cwd);
const pages = mergeResults(existing.pages, successful);
await updatePageIndex({ pages }, cwd);
}
return results;
}
export async function listStoredPages(cwd = process.cwd()) {
const sources = await listSources(cwd);
return sources.pages;
}
//# sourceMappingURL=api.js.map
+1
dist/api.js.map
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,GAGlB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,WAAW,EACX,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAe5C,SAAS,YAAY,CAAC,QAAqB,EAAE,OAAsB;IACjE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAE7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAc;YACvB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,GAAG;YACd,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG;YAC5B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,eAAe;YAClD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,IAAI,aAAqB,CAAC;IAE1B,IAAI,CAAC;QACH,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,GAAG,EAAE,IAAI;YACT,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjE,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,CAAC,WAAW,IAAI,aAAa,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE9E,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,MAAM,EAAE,QAAQ,CAAC,MAAM;SACxB,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnD,OAAO,QAAQ,CAAC,QAAQ,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UAA6B,EAAE;IAE/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;IAEtC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAc,EACd,UAA6B,EAAE;IAE/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,OAAO,CAAC,KAAK,CAAC;AACvB,CAAC"}
+2
dist/api.test.d.ts
export {};
//# sourceMappingURL=api.test.d.ts.map
+1
dist/api.test.d.ts.map
{"version":3,"file":"api.test.d.ts","sourceRoot":"","sources":["../src/api.test.ts"],"names":[],"mappings":""}
+89
dist/api.test.js
import { describe, it, expect } from "vitest";
import { mkdtemp, readFile, rm } from "fs/promises";
import { join } from "path";
import { tmpdir } from "os";
import { fetchToStore, listStoredPages, fetchManyToStore, } from "./api.js";
import { fetchMarkdown, fetchRawMarkdown } from "./web.js";
describe("api", () => {
it("fetchMarkdown normalizes URL before requesting", async () => {
let requestedUrl = "";
const mockFetch = async (input) => {
requestedUrl = String(input);
return new Response("# Hello", {
status: 200,
headers: {
"content-type": "text/markdown; charset=utf-8",
},
});
};
const result = await fetchMarkdown("example.com/docs", {
fetchImpl: mockFetch,
});
expect(requestedUrl).toBe("https://example.com/docs");
expect(result.markdown).toBe("# Hello");
});
it("fetchRawMarkdown returns markdown content only", async () => {
const mockFetch = async () => new Response("# Raw", {
status: 200,
headers: {
"content-type": "text/markdown; charset=utf-8",
},
});
const markdown = await fetchRawMarkdown("example.com", {
fetchImpl: mockFetch,
});
expect(markdown).toBe("# Raw");
});
it("fetchToStore writes snapshot and updates sources index", async () => {
const cwd = await mkdtemp(join(tmpdir(), "mdrip-api-"));
try {
const mockFetch = async () => new Response("# Stored", {
status: 200,
headers: {
"content-type": "text/markdown; charset=utf-8",
"x-markdown-tokens": "2",
},
});
const result = await fetchToStore("https://example.com/docs/page", {
cwd,
fetchImpl: mockFetch,
});
expect(result.success).toBe(true);
expect(result.path).toBe("pages/example.com/docs/page.md");
const pages = await listStoredPages(cwd);
expect(pages).toHaveLength(1);
expect(pages[0].url).toBe("https://example.com/docs/page");
const filePath = join(cwd, "mdrip", result.path);
const content = await readFile(filePath, "utf-8");
expect(content).toBe("# Stored");
}
finally {
await rm(cwd, { recursive: true, force: true });
}
});
it("fetchManyToStore records mixed success results", async () => {
const cwd = await mkdtemp(join(tmpdir(), "mdrip-api-"));
try {
const mockFetch = async () => new Response("# Multi", {
status: 200,
headers: {
"content-type": "text/markdown; charset=utf-8",
},
});
const results = await fetchManyToStore(["https://example.com/a", "ftp://example.com/a"], {
cwd,
fetchImpl: mockFetch,
});
expect(results).toHaveLength(2);
expect(results[0].success).toBe(true);
expect(results[1].success).toBe(false);
const pages = await listStoredPages(cwd);
expect(pages).toHaveLength(1);
expect(pages[0].url).toBe("https://example.com/a");
}
finally {
await rm(cwd, { recursive: true, force: true });
}
});
});
//# sourceMappingURL=api.test.js.map
+1
dist/api.test.js.map
{"version":3,"file":"api.test.js","sourceRoot":"","sources":["../src/api.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,eAAe,EACf,gBAAgB,GACjB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE3D,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE;IACnB,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,MAAM,SAAS,GAAiB,KAAK,EAAE,KAAK,EAAE,EAAE;YAC9C,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,OAAO,IAAI,QAAQ,CAAC,SAAS,EAAE;gBAC7B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,8BAA8B;iBAC/C;aACF,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE;YACrD,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACtD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,OAAO,EAAE;YACpB,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,8BAA8B;aAC/C;SACF,CAAC,CAAC;QAEL,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE;YACrD,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACvB,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,8BAA8B;oBAC9C,mBAAmB,EAAE,GAAG;iBACzB;aACF,CAAC,CAAC;YAEL,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,+BAA+B,EAAE;gBACjE,GAAG;gBACH,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YAE3D,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAE3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,SAAS,EAAE;gBACtB,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,8BAA8B;iBAC/C;aACF,CAAC,CAAC;YAEL,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,EAChD;gBACE,GAAG;gBACH,SAAS,EAAE,SAAS;aACrB,CACF,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEvC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrD,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+5
dist/web.d.ts
import { type FetchMarkdownOptions, type MarkdownResponse } from "./lib/cloudflare.js";
export type { FetchMarkdownOptions, MarkdownResponse };
export declare function fetchMarkdown(url: string, options?: FetchMarkdownOptions): Promise<MarkdownResponse>;
export declare function fetchRawMarkdown(url: string, options?: FetchMarkdownOptions): Promise<string>;
//# sourceMappingURL=web.d.ts.map
+1
dist/web.d.ts.map
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,CAAC;AAEvD,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CAG3B;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CAGjB"}
+11
dist/web.js
import { fetchMarkdownPage, } from "./lib/cloudflare.js";
import { normalizeUrl } from "./lib/url.js";
export async function fetchMarkdown(url, options = {}) {
const normalizedUrl = normalizeUrl(url);
return fetchMarkdownPage(normalizedUrl, options);
}
export async function fetchRawMarkdown(url, options = {}) {
const response = await fetchMarkdown(url, options);
return response.markdown;
}
//# sourceMappingURL=web.js.map
+1
dist/web.js.map
{"version":3,"file":"web.js","sourceRoot":"","sources":["../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,GAGlB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnD,OAAO,QAAQ,CAAC,QAAQ,CAAC;AAC3B,CAAC"}
+1
-1
dist/index.js

@@ -11,3 +11,3 @@ #!/usr/bin/env node

.description("Fetch markdown snapshots for URLs using Cloudflare Markdown for Agents")
.version("0.1.2")
.version("0.1.3")
.option("--cwd <path>", "working directory (default: current directory)");

@@ -14,0 +14,0 @@ program

+1
-1
dist/lib/gitignore.d.ts.map

@@ -1,1 +0,1 @@

{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAOA,wBAAsB,aAAa,CACjC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAkBlB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB"}
{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAOA,wBAAsB,aAAa,CACjC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB"}
+5
-2
dist/lib/gitignore.js
import { readFile, writeFile } from "fs/promises";
import { join } from "path";
import { existsSync } from "fs";
const MDRIP_ENTRY = "mdrip/";
const MDRIP_ENTRY = "/mdrip/";
const MARKER_COMMENT = "# mdrip - markdown snapshots for agents";

@@ -16,3 +16,6 @@ export async function hasMdripEntry(cwd = process.cwd()) {

const trimmed = line.trim();
return trimmed === MDRIP_ENTRY || trimmed === "mdrip";
return (trimmed === MDRIP_ENTRY ||
trimmed === "mdrip/" ||
trimmed === "/mdrip" ||
trimmed === "mdrip");
});

@@ -19,0 +22,0 @@ }

+1
-1
dist/lib/gitignore.js.map

@@ -1,1 +0,1 @@

{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,MAAM,WAAW,GAAG,QAAQ,CAAC;AAC7B,MAAM,cAAc,GAAG,yCAAyC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,GAAG,cAAc,KAAK,WAAW,IAAI,CAAC;IAEjD,MAAM,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC"}
{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,MAAM,WAAW,GAAG,SAAS,CAAC;AAC9B,MAAM,cAAc,GAAG,yCAAyC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,CACL,OAAO,KAAK,WAAW;gBACvB,OAAO,KAAK,QAAQ;gBACpB,OAAO,KAAK,QAAQ;gBACpB,OAAO,KAAK,OAAO,CACpB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,GAAG,cAAc,KAAK,WAAW,IAAI,CAAC;IAEjD,MAAM,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC"}
+17
-1
package.json
{
"name": "mdrip",
"version": "0.1.2",
"version": "0.1.3",
"description": "Fetch markdown snapshots of web pages using Cloudflare Markdown for Agents",
"type": "module",
"main": "./dist/web.js",
"types": "./dist/web.d.ts",
"exports": {
".": {
"types": "./dist/web.d.ts",
"import": "./dist/web.js"
},
"./node": {
"types": "./dist/api.d.ts",
"import": "./dist/api.js"
},
"./cli": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"bin": {

@@ -7,0 +23,0 @@ "mdrip": "./dist/index.js"

+42
-0
README.md

@@ -61,2 +61,44 @@ # mdrip

For programmatic usage in Node.js or Workers:
```bash
npm install mdrip
```
## Programmatic API
### Node.js (fetch and store)
```ts
import { fetchToStore, listStoredPages } from "mdrip/node";
const result = await fetchToStore("https://developers.cloudflare.com/", {
cwd: process.cwd(),
});
if (!result.success) {
throw new Error(result.error || "Failed to fetch page");
}
const pages = await listStoredPages(process.cwd());
console.log(pages.map((p) => p.path));
```
### Cloudflare Workers / Agent runtimes (raw in-memory markdown)
```ts
import { fetchMarkdown } from "mdrip";
const page = await fetchMarkdown(
"https://blog.cloudflare.com/markdown-for-agents/",
);
console.log(page.markdownTokens);
console.log(page.markdown);
```
Available programmatic methods:
- `mdrip` (Workers-safe): `fetchMarkdown(url, options)`, `fetchRawMarkdown(url, options)`
- `mdrip/node` (filesystem features): `fetchToStore(url, options)`, `fetchManyToStore(urls, options)`, `listStoredPages(cwd?)`
## Usage

@@ -63,0 +105,0 @@