🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

crawlex

Package Overview
Dependencies
Maintainers
1
Versions
56
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

crawlex - npm Package Compare versions

Comparing version
1.0.4
 to
1.0.6-next.121
+345
sdk/shell.js
'use strict';
// crawlex shell (slice 23) — Node REPL with crawlex SDK preloaded.
//
// Mirrors the Rust shell (slice 22) helpers: `fetch`, `css`, `xpath`,
// `findByText`, `findByRegex`, `save`. Implementation is intentionally
// minimal — pure Node, no extra deps:
// * fetch: node:http(s) GET
// * selectors: regex-based subset (tag / tag#id / tag.class, //tag)
// * adaptive save: JSON file under $XDG_DATA_HOME/crawlex
//
// Two layers (mirrors the Rust shell):
// * `createShellApi` is the test surface — returns the helper object
// and shared state without spinning up a REPL. Tests inject a stub
// fetcher and assert on the state mutations.
// * `startRepl` wires the API into `node:repl` with history.
const repl = require('node:repl');
const path = require('node:path');
const fs = require('node:fs');
const os = require('node:os');
const http = require('node:http');
const https = require('node:https');
const { URL } = require('node:url');
function xdgDataHome() {
return process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
}
function defaultHistoryPath() {
return path.join(xdgDataHome(), 'crawlex', 'shell_history_node');
}
function defaultStorePath() {
return path.join(xdgDataHome(), 'crawlex', 'adaptive_store.json');
}
function httpFetch(url) {
const u = new URL(url);
const mod = u.protocol === 'https:' ? https : http;
return new Promise((resolve, reject) => {
const req = mod.request(
{
method: 'GET',
hostname: u.hostname,
port: u.port || (u.protocol === 'https:' ? 443 : 80),
path: u.pathname + u.search,
headers: { 'user-agent': 'crawlex-shell' },
},
(res) => {
const chunks = [];
res.on('data', (c) => chunks.push(c));
res.on('end', () => {
resolve({
finalUrl: url,
status: res.statusCode || 0,
headers: res.headers,
body: Buffer.concat(chunks).toString('utf8'),
});
});
},
);
req.on('error', reject);
req.end();
});
}
function parseSimpleSelector(sel) {
// Supports a minimal subset: tag, #id, .class, tag#id, tag.class.
// Anything richer (descendant combinators, attribute selectors) is
// out-of-scope for the JS shell — the Rust shell remains the heavy
// engine.
const m = /^([a-zA-Z*][a-zA-Z0-9-]*)?(?:#([a-zA-Z0-9_-]+))?(?:\.([a-zA-Z0-9_-]+))?$/.exec(
sel.trim(),
);
if (!m || (!m[1] && !m[2] && !m[3])) return null;
return { tag: m[1] || '*', id: m[2] || null, class: m[3] || null };
}
function parseAttrs(s) {
const out = {};
const re = /\b([a-zA-Z][a-zA-Z0-9-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g;
let m;
while ((m = re.exec(s))) {
out[m[1]] = m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3] : m[4];
}
return out;
}
const VOID_TAGS = new Set([
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
'link', 'meta', 'param', 'source', 'track', 'wbr',
]);
// Walk every element in the document, including nested ones, yielding
// `{ tag, attrs, inner, outer, index }`. The regex iterator approach
// (matchAll on `<x>...</x>`) only catches top-level matches because the
// engine advances past each match. Here we re-scan from every `<tag>`
// open and resolve the matching `</tag>` with a depth counter so nested
// elements with the same tag name (e.g. `<div>` inside `<div>`) close
// correctly. O(n*k) for k opens — fine for shell-sized pages.
function* elementIter(html) {
const openRe = /<([a-zA-Z][a-zA-Z0-9-]*)\b([^>]*)>/g;
let m;
while ((m = openRe.exec(html))) {
const tag = m[1];
const attrs = m[2];
if (VOID_TAGS.has(tag.toLowerCase())) continue;
if (/\/\s*$/.test(attrs)) continue; // self-closing like <br/>
const startInner = m.index + m[0].length;
const close = findMatchingClose(html, tag, startInner);
if (close < 0) continue;
yield {
0: html.slice(m.index, close + tag.length + 3),
1: tag,
2: attrs,
3: html.slice(startInner, close),
index: m.index,
};
}
}
function findMatchingClose(html, tag, from) {
const openRe = new RegExp(`<${tag}\\b[^>]*>`, 'gi');
const closeRe = new RegExp(`</${tag}\\s*>`, 'gi');
let depth = 1;
let pos = from;
while (depth > 0) {
openRe.lastIndex = pos;
closeRe.lastIndex = pos;
const o = openRe.exec(html);
const c = closeRe.exec(html);
if (!c) return -1;
if (o && !/\/\s*$/.test(o[0].slice(1, -1)) && o.index < c.index) {
depth += 1;
pos = o.index + o[0].length;
} else {
depth -= 1;
if (depth === 0) return c.index;
pos = c.index + c[0].length;
}
}
return -1;
}
function textOf(html) {
return html
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
.replace(/<[^>]+>/g, ' ')
.replace(/&nbsp;/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
function cssQuery(html, selector) {
const parsed = parseSimpleSelector(selector);
if (!parsed) {
throw new Error(
`unsupported selector '${selector}'. JS shell supports tag, #id, .class, tag#id, tag.class.`,
);
}
const out = [];
for (const m of elementIter(html)) {
const tag = m[1];
if (parsed.tag !== '*' && tag.toLowerCase() !== parsed.tag.toLowerCase()) continue;
const attrs = parseAttrs(m[2]);
if (parsed.id && attrs.id !== parsed.id) continue;
if (parsed.class) {
const classes = (attrs.class || '').split(/\s+/);
if (!classes.includes(parsed.class)) continue;
}
out.push({ tag, attrs, inner: m[3], outer: m[0], text: textOf(m[3]) });
}
return out;
}
function xpathQuery(html, expr) {
// Supports `//tag` only. Anything richer goes to the Rust shell.
const m = /^\/\/([a-zA-Z][a-zA-Z0-9-]*)$/.exec(expr.trim());
if (!m) {
throw new Error(`unsupported xpath '${expr}'. JS shell supports //tag only.`);
}
return cssQuery(html, m[1]);
}
function findByText(html, needle) {
if (typeof needle !== 'string' || needle.length === 0) {
throw new TypeError('findByText: needle must be a non-empty string');
}
const out = [];
for (const m of elementIter(html)) {
const text = textOf(m[3]);
if (text.includes(needle)) {
out.push({ tag: m[1], attrs: parseAttrs(m[2]), inner: m[3], outer: m[0], text });
}
}
return out;
}
function findByRegex(html, pattern) {
const re = pattern instanceof RegExp ? pattern : new RegExp(pattern);
const out = [];
for (const m of elementIter(html)) {
const text = textOf(m[3]);
if (re.test(text)) {
out.push({ tag: m[1], attrs: parseAttrs(m[2]), inner: m[3], outer: m[0], text });
}
}
return out;
}
class Page {
constructor(state, resp) {
this._state = state;
this.url = resp.finalUrl;
this.status = resp.status;
this.headers = resp.headers;
this.body = resp.body;
}
_record(result, query) {
this._state.lastSelection = result[0] || null;
this._state.lastSelectionQuery = query;
this._state.lastSelectionUrl = this.url;
return result;
}
css(selector) {
return this._record(cssQuery(this.body, selector), `css:${selector}`);
}
xpath(expr) {
return this._record(xpathQuery(this.body, expr), `xpath:${expr}`);
}
findByText(needle) {
return this._record(findByText(this.body, needle), `text:${needle}`);
}
findByRegex(pattern) {
const label = pattern instanceof RegExp ? pattern.source : String(pattern);
return this._record(findByRegex(this.body, pattern), `regex:${label}`);
}
save(identifier) {
if (typeof identifier !== 'string' || !identifier) {
throw new TypeError('save: identifier must be a non-empty string');
}
if (!this._state.lastSelection) {
throw new Error('save: no element selected. Run css/xpath/findByText first.');
}
const host = new URL(this.url).host;
const storePath = this._state.storePath;
let store = {};
try {
store = JSON.parse(fs.readFileSync(storePath, 'utf8'));
} catch (_) {
store = {};
}
const entry = {
query: this._state.lastSelectionQuery,
tag: this._state.lastSelection.tag,
attrs: this._state.lastSelection.attrs,
saved_at: new Date().toISOString(),
};
store[host] = store[host] || {};
store[host][identifier] = entry;
fs.mkdirSync(path.dirname(storePath), { recursive: true });
fs.writeFileSync(storePath, JSON.stringify(store, null, 2));
return entry;
}
}
function createShellApi(opts = {}) {
const fetcher = opts.fetcher || httpFetch;
const state = {
last: null,
lastSelection: null,
lastSelectionQuery: null,
lastSelectionUrl: null,
storePath: opts.storePath || defaultStorePath(),
};
const crawlex = {
async fetch(url) {
if (typeof url !== 'string' || !url) {
throw new TypeError('crawlex.fetch: url must be a non-empty string');
}
const resp = await fetcher(url);
const page = new Page(state, resp);
state.last = page;
return page;
},
css(sel) { return requireLast(state).css(sel); },
xpath(e) { return requireLast(state).xpath(e); },
findByText(t) { return requireLast(state).findByText(t); },
findByRegex(p) { return requireLast(state).findByRegex(p); },
save(id) { return requireLast(state).save(id); },
get last() { return state.last; },
};
return { crawlex, state };
}
function requireLast(state) {
if (!state.last) {
throw new Error('no page fetched yet — call crawlex.fetch(url) first.');
}
return state.last;
}
function startRepl(opts = {}) {
const stdin = opts.stdin || process.stdin;
const stdout = opts.stdout || process.stdout;
const historyPath = opts.historyPath || defaultHistoryPath();
const { crawlex } = createShellApi({ storePath: opts.storePath });
stdout.write(`crawlex shell — node ${process.version}\n`);
stdout.write(
`crawlex globals: fetch(url), css(sel), xpath(expr), findByText(s), findByRegex(re), save(id), last\n`,
);
stdout.write(`type .exit to quit.\n`);
const server = repl.start({
prompt: 'crawlex> ',
input: stdin,
output: stdout,
useColors: stdout.isTTY === true,
});
server.context.crawlex = crawlex;
Object.defineProperty(server.context, 'last', {
get: () => crawlex.last,
configurable: true,
});
try {
fs.mkdirSync(path.dirname(historyPath), { recursive: true });
server.setupHistory(historyPath, () => {});
} catch (_) {
// History is best-effort — non-fatal if the data dir is unwritable.
}
return server;
}
module.exports = {
createShellApi,
startRepl,
Page,
cssQuery,
xpathQuery,
findByText,
findByRegex,
defaultHistoryPath,
defaultStorePath,
};
+2
-1
package.json
{
"name": "crawlex",
"version": "1.0.4",
"version": "1.0.6-next.121",
"description": "Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue",

@@ -21,2 +21,3 @@ "type": "commonjs",

"sdk/crawlex-sdk.js",
"sdk/shell.js",
"sdk/postinstall.js",

@@ -23,0 +24,0 @@ "sdk/index.d.ts",

+66
-21
README.md

@@ -38,3 +38,3 @@ <div align="center">

<tr><td>📦 <strong>Catalog</strong></td><td>30 Chrome stable × 30 Chromium × 20 Firefox × Edge × Safari fingerprints. Era-fallback resolution: ask for <code>chrome-149-linux</code>, get the closest captured profile</td></tr>
<tr><td>🛠️ <strong>Worker scope</strong></td><td>Same shim auto-attached to dedicated / shared / service workers via CDP <code>Target.setAutoAttach</code> — Camoufox port</td></tr>
<tr><td>🛠️ <strong>Worker scope</strong></td><td>Same shim auto-attached to dedicated / shared / service workers via CDP <code>Target.setAutoAttach</code></td></tr>
</table>

@@ -63,2 +63,13 @@

## 🆕 Last 24h highlights
- `1.0.6` release line focuses on canonical URL identity, lossless frontier admission and cache/queue alias deduplication.
- JS/TS hooks now run through the SDK bridge, so `defineHooks()` can drive the same lifecycle decisions as embedded Rust hooks.
- NDJSON events now carry richer artifacts, Web Vitals, per-fetch timings, crawl-attempt telemetry and crawl-resolution summaries.
- The supported release artifact is the full `crawlex` binary with RedDB persistence enabled.
- Large crawl efficiency grew: cache validation, prefetch discovery mode and best-first URL scoring are now available from CLI/config.
- Render fallback grew: external CDP connection, GPU posture control, Shadow DOM flattening, overlay cleanup and last-resort fallback fetch are configurable.
---
## 🏃 Quickstart

@@ -197,6 +208,6 @@

crtsh: true, // certificate-transparency seeding
storage: 'sqlite',
storagePath: './crawl.db',
queue: 'sqlite',
queuePath: './crawl.db',
storage: 'reddb',
storagePath: 'file://./state/crawl.rdb',
queue: 'reddb',
queuePath: 'file://./state/frontier.rdb',
proxies: ['http://user:pass@proxy1:8080', 'http://user:pass@proxy2:8080'],

@@ -299,2 +310,21 @@ proxyStrategy: 'health-weighted',

### 6. Large crawl: validate cache, prefetch links, score the frontier
```bash
crawlex pages run \
--seed https://docs.example.com \
--method auto \
--queue reddb --queue-path file://./state/frontier.rdb \
--storage reddb --storage-path file://./state/crawl.rdb \
--cache-validate \
--cache-max-age-secs 86400 \
--prefetch \
--best-first \
--score-keyword docs \
--score-keyword api \
--emit ndjson
```
This mode is for discovery passes: reuse fresh cache rows, harvest links cheaply, and let higher-value URLs rise in the queue before expensive render passes.
---

@@ -328,2 +358,4 @@

- 🧬 Asset-ref classification (JS / CSS / image / API / nav)
- ⚡ Prefetch mode for fast discovery-only passes
- 🎯 Best-first URL scoring with keyword bonuses
- 🔓 TCP port scan (opt-in, network-active)

@@ -335,2 +367,4 @@

- 🔄 Policy decisions: keep / drop / retry / scope-demote / proxy-rotate / give-up
- 🧱 Unified block classifier with attempt-level crawl stats
- 🪂 Fallback fetch command for last-resort HTML retrieval
- 🎯 4 captcha solver adapters: in-house reCAPTCHA v3, 2captcha, anticaptcha, VLM

@@ -343,4 +377,8 @@

- 🎯 Render pool — Chromium auto-fetch + isolated user-data dirs
- 🔁 Persistent queue: in-memory / SQLite / Redis backends
- 💾 Storage: filesystem / SQLite / memory — opt-in per concern (artifact, state, challenge, telemetry, intel)
- 🔌 External CDP endpoint support for managed/browser-farm Chrome
- 🌑 Shadow DOM flattening + overlay / consent-popup cleanup
- 🖥️ GPU policy: compatibility mode or stealth-friendly GPU surfaces
- 🔁 Queue: RedDB-native durable frontier by default; memory remains only for isolated smokes/tests
- 💾 Storage: RedDB for crawl state, telemetry and intel; filesystem remains available for large artifacts
- 🧠 Smart cache validation: `ETag`, `Last-Modified`, `<head>` fingerprint
- 🔄 Proxy rotator — health checks + sticky sessions + per-host affinity

@@ -354,5 +392,6 @@ - 📊 Web Vitals + per-fetch network breakdown (DNS / TCP / TLS / TTFB / download)

- 📜 NDJSON event stream — versioned envelope (`v: 1`)
- 🎬 19 event kinds covering full lifecycle
- 🎬 21 event kinds covering full lifecycle
- 🔬 Embedded `WebVitals` summary on `render.completed`
- ⏱️ Per-request timings on `fetch.completed` (ALPN, cipher, TLS version)
- 🧾 `crawl.attempted` / `crawl.resolved` telemetry for HTTP → render → fallback ladders
- 📸 Artifact descriptors with on-disk path on the wire

@@ -367,6 +406,7 @@ - 🪝 Hooks: 12 lifecycle points × 3 languages (Rust / JS / Lua)

- 🔌 SDK `crawl()` async iterator
- 🧩 SDK `defineHooks()` bridge for JS/TS lifecycle hooks
- 📚 docsify docs site (GitHub Pages)
- 🧪 386+ lib tests, 27 fpjs compliance, TLS catalog roundtrip suite
- 🧪 820+ lib tests, fpjs compliance, TLS catalog roundtrip suite
- 🔐 Optional Lua hooks (`mlua`)
- 🪶 Two binaries: `crawlex` (full) + `crawlex-mini` (HTTP-only, no Chromium)
- 🪶 Single supported release binary: `crawlex` with HTTP impersonation, Chromium rendering and RedDB persistence

@@ -381,3 +421,3 @@ </td>

Every run emits one JSON envelope per line on stdout. Versioned, stable, 19 kinds:
Every run emits one JSON envelope per line on stdout. Versioned, stable, 21 kinds:

@@ -388,2 +428,3 @@ ```jsonl

{"v":1,"event":"fetch.completed","run_id":42,"url":"https://target.com/","data":{"final_url":"https://target.com/","status":200,"bytes":98234,"body_truncated":false,"dns_ms":12,"tcp_connect_ms":18,"tls_handshake_ms":24,"ttfb_ms":142,"download_ms":83,"total_ms":280,"alpn":"h2","tls_version":"TLSv1.3","cipher":"TLS_AES_128_GCM_SHA256"}}
{"v":1,"event":"crawl.attempted","run_id":42,"url":"https://target.com/","data":{"crawl_id":42,"attempt_index":1,"engine":"http_spoof","status":403,"blocked":true,"block_reason":"Cloudflare challenge form"}}
{"v":1,"event":"render.completed","run_id":42,"session_id":"sess_abc","url":"https://target.com/","data":{"final_url":"https://target.com/","status":200,"manifest":true,"service_workers":1,"is_spa":true,"vitals":{"ttfb_ms":142,"first_contentful_paint_ms":380.5,"largest_contentful_paint_ms":920.1,"cumulative_layout_shift":0.03,"total_blocking_time_ms":50.0,"dom_nodes":1842,"js_heap_used_bytes":12345678,"resource_count":45,"total_transfer_bytes":982341}}}

@@ -393,2 +434,3 @@ {"v":1,"event":"artifact.saved","run_id":42,"url":"https://target.com/","data":{"kind":"screenshot.full_page","mime":"image/png","size":1234567,"sha256":"a1b2c3...","path":"artifacts/sess_abc/1714123456_screenshot_full_page_a1b2c3d4.png"}}

{"v":1,"event":"decision.made","run_id":42,"url":"https://protected.com/","why":"render:js-challenge","data":{"decision":"retry","reason":{"code":"render:js-challenge"}}}
{"v":1,"event":"crawl.resolved","run_id":42,"url":"https://target.com/","data":{"crawl_id":42,"attempts_count":2,"fallback_fetch_used":false,"resolved_by":"render","success":true}}
{"v":1,"event":"run.completed","run_id":42}

@@ -444,3 +486,6 @@ ```

Q --> P[Policy Engine]
P -->|http| F[ImpersonateClient<br/>BoringSSL + h2 patched]
P --> C[Cache Validator<br/>ETag + Last-Modified + head fingerprint]
C -->|fresh| ST[Storage<br/>5 traits]
C -->|stale| F[ImpersonateClient<br/>BoringSSL + h2 patched]
P -->|http| F
P -->|render| R[RenderPool<br/>Chromium + stealth shim]

@@ -450,5 +495,5 @@ F --> X[Extractor<br/>+ Asset Refs]

X --> D[Discovery<br/>Pipeline]
X --> ST[Storage<br/>5 traits]
X --> ST
D --> Q
P --> EV[NDJSON Events<br/>19 kinds]
P --> EV[NDJSON Events<br/>21 kinds]
R --> H1[Rust Hooks]

@@ -465,2 +510,3 @@ R --> H2[JS Bridge]

- `antibot/` — vendor classifier + 4 captcha solver adapters
- `cache_validator/` — cache freshness by HTTP validators and head fingerprints
- `storage/` — 5 concern-oriented traits (artifact / state / challenge / telemetry / intel)

@@ -480,3 +526,3 @@ - `events/` — NDJSON envelope + sink (stdout / null / memory)

| Async | tokio multi-thread |
| Storage | rusqlite (SQLite WAL), DashMap (memory), filesystem layout |
| Storage / Queue | RedDB embedded/client storage plus RedDB Queue frontier; DashMap memory only for isolated tests |
| Discovery | hickory-resolver (DNS), reqwest (RDAP), texting_robots (robots.txt) |

@@ -486,5 +532,4 @@ | Lua | mlua 0.10 (optional, `lua-hooks` feature) |

**Two binaries** ship from one source tree:
- `crawlex` — **full** build with HTTP impersonation + Chromium rendering + stealth shim + persistent queue
- `crawlex-mini` — **HTTP-only** worker, no Chromium dependency, same CLI surface (browser-only flags return `Error::RenderDisabled`)
**Release binary:**
- `crawlex` — full build with HTTP impersonation + Chromium rendering + stealth shim + RedDB persistence

@@ -501,4 +546,4 @@ ---

| Worker-scope stealth | ✅ auto-attach | ⚠️ manual | ⚠️ manual | ❌ |
| HTTP-only path (no browser) | ✅ `crawlex-mini` | ❌ | ❌ | ✅ |
| Persistent queue + resume | ✅ SQLite/Redis | ❌ external | ❌ external | ❌ |
| HTTP-only path (no browser) | ✅ full binary policy path | ❌ | ❌ | ✅ |
| RedDB telemetry + resume state | ✅ native | ❌ external | ❌ external | ❌ |
| Discovery pipeline | ✅ 17 stages | ❌ | ❌ | ❌ |

@@ -530,3 +575,3 @@ | Streaming NDJSON events | ✅ versioned | ❌ | ❌ | ❌ |

# Unit tests + offline shim compliance
cargo test --lib # 386+ tests
cargo test --lib # 820+ tests
cargo test --test fpjs_compliance # 27 cases

@@ -533,0 +578,0 @@ cargo test --test tls_catalog_coverage --test tls_catalog_roundtrip

+366
-0
sdk/crawlex-sdk.js

@@ -387,2 +387,10 @@ #!/usr/bin/env node

function runCli(argv) {
// Slice 23 — `crawlex shell` is a Node REPL handled in JS, not a
// passthrough to the rust binary. Match only the bare `shell`
// subcommand; flags like `--help` after `shell` still want help text
// from the native binary if it ever grows one.
if (argv[0] === 'shell' && argv.length === 1) {
require('./shell.js').startRepl();
return;
}
const bin = binaryPath();

@@ -548,2 +556,356 @@ if (!fs.existsSync(bin) && !process.env.CRAWLEX_FORCE_BINARY) {

/**
* Async iterator over `crawlex pages list` results (slice 8). Hides
* the cursor token from the caller — internally re-invokes the binary
* once per page using the prior response's `next_cursor` and
* terminates when the server stops returning one.
*/
async function* paginatePages(opts) {
if (!opts || typeof opts.storagePath !== 'string' || !opts.storagePath) {
throw new TypeError('paginatePages: opts.storagePath is required');
}
const pageSize = opts.pageSize == null ? 100 : Number(opts.pageSize);
if (!Number.isInteger(pageSize) || pageSize <= 0) {
throw new RangeError(`paginatePages: pageSize must be a positive integer (got ${opts.pageSize})`);
}
const runOpts = {};
if (opts.bin) runOpts.bin = opts.bin;
if (opts.env) runOpts.env = opts.env;
let cursor = null;
while (true) {
const argv = ['pages', 'list', '--storage-path', opts.storagePath, '--limit', String(pageSize)];
if (opts.status) argv.push('--status', opts.status);
if (cursor) argv.push('--cursor', cursor);
const resp = runJson(argv, runOpts);
if (!resp || !Array.isArray(resp.rows)) {
throw new Error('paginatePages: unexpected response shape from `pages list`');
}
for (const row of resp.rows) yield row;
if (!resp.next_cursor) return;
cursor = resp.next_cursor;
}
}
// --- v2 scraping framework: Request (slice 16) ------------------------
// Pure JS placeholder. Runtime dispatch against the Rust SessionManager
// lands once the engine bindings ship; the class exists today so recipes
// can already author against the documented shape.
class Request {
constructor(url, opts = {}) {
if (typeof url !== 'string' || url.length === 0) {
throw new TypeError('Request: url must be a non-empty string');
}
this.url = url;
this.method = typeof opts.method === 'string' ? opts.method : 'GET';
if (opts.sessionId !== undefined) {
if (typeof opts.sessionId !== 'string' || opts.sessionId.length === 0) {
throw new TypeError('Request: sessionId must be a non-empty string');
}
this.sessionId = opts.sessionId;
}
}
}
// --- v2 scraping framework: defineSpider / runSpider (slice 17) ------
// Pure JS spider runtime: the Rust `SpiderRunner` is the production
// driver, but a self-contained JS implementation is shipped here so
// recipes can run end-to-end against a Node http(s) fetcher today.
// Both share the same Checkpoint wire shape (see `src/scraping/spider.rs`)
// so a JS run can pause and resume in Rust (or vice versa) once the
// engine binding lands.
function defineSpider(spec = {}) {
if (!spec || typeof spec !== 'object') {
throw new TypeError('defineSpider: spec must be an object');
}
if (!Array.isArray(spec.startUrls) || spec.startUrls.length === 0) {
throw new TypeError('defineSpider: startUrls must be a non-empty array');
}
if (typeof spec.parse !== 'function') {
throw new TypeError('defineSpider: parse must be a function');
}
return {
startUrls: spec.startUrls.slice(),
parse: spec.parse,
downloadDelayMs: Number(spec.downloadDelayMs) || 0,
robotsTxtObey: Boolean(spec.robotsTxtObey),
userAgent: typeof spec.userAgent === 'string' ? spec.userAgent : 'crawlex',
maxItems: Number.isInteger(spec.maxItems) && spec.maxItems > 0 ? spec.maxItems : null,
};
}
function reqKey(r) {
return `${r.method || 'GET'} ${r.url}`;
}
function hostOf(url) {
try { return new URL(url).host; } catch (_) { return ''; }
}
// Minimal robots.txt evaluator — Disallow + optional Crawl-delay only.
// Honours UA-specific blocks falling back to `*`. Anything beyond what
// slice 17 promises (no Allow precedence, no path globs beyond simple
// prefix) is intentionally deferred to the Rust dispatcher.
function evaluateRobots(body, userAgent, urlPath) {
if (!body) return { allowed: true, crawlDelayMs: 0 };
const uaLc = userAgent.toLowerCase();
let currentUAs = [];
let bestSpecificity = -1; // -1 none, 0 wildcard, 1 exact
let allowed = true;
let crawlDelayMs = 0;
let activeBlock = false;
for (const raw of body.split(/\r?\n/)) {
const line = raw.replace(/#.*$/, '').trim();
if (!line) {
currentUAs = [];
continue;
}
const m = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
if (!m) continue;
const key = m[1].toLowerCase();
const val = m[2].trim();
if (key === 'user-agent') {
currentUAs.push(val.toLowerCase());
const spec = currentUAs.includes(uaLc) ? 1 : currentUAs.includes('*') ? 0 : -1;
activeBlock = spec >= bestSpecificity;
if (spec > bestSpecificity) {
bestSpecificity = spec;
allowed = true;
crawlDelayMs = 0;
}
continue;
}
if (!activeBlock) continue;
if (key === 'disallow') {
if (val && urlPath.startsWith(val)) allowed = false;
} else if (key === 'crawl-delay') {
const n = Number(val);
if (Number.isFinite(n)) crawlDelayMs = Math.max(crawlDelayMs, Math.round(n * 1000));
}
}
return { allowed, crawlDelayMs };
}
function sleep(ms) {
return ms > 0 ? new Promise((r) => setTimeout(r, ms)) : Promise.resolve();
}
async function defaultFetcher(req) {
const http = require('node:http');
const https = require('node:https');
const url = new URL(req.url);
const mod = url.protocol === 'https:' ? https : http;
return await new Promise((resolve, reject) => {
const r = mod.request(
{
method: req.method || 'GET',
hostname: url.hostname,
port: url.port || (url.protocol === 'https:' ? 443 : 80),
path: url.pathname + url.search,
headers: { 'user-agent': req.userAgent || 'crawlex' },
},
(res) => {
const chunks = [];
res.on('data', (c) => chunks.push(c));
res.on('end', () => {
resolve({
request: req,
finalUrl: req.url,
status: res.statusCode || 0,
headers: res.headers,
body: Buffer.concat(chunks),
});
});
},
);
r.on('error', reject);
r.end();
});
}
/**
* Drive a spider built by [`defineSpider`]. Async generator yielding
* each emitted item. Honours `AbortSignal` (Ctrl-C path) and returns a
* resumable checkpoint via `handle.checkpoint()` (call after `for await`
* loop exits).
*/
function runSpider(spider, opts = {}) {
const fetcher = opts.fetcher || defaultFetcher;
const robotsCache = opts.robotsCache || new Map(); // host -> body
const signal = opts.signal;
const seen = new Set();
const pending = [];
const lastFetchPerHost = new Map();
let itemsEmitted = 0;
let paused = false;
// Bounded broadcaster for `stream()` consumers. Each subscriber holds
// its own ring buffer; overflow drops the oldest item rather than
// blocking the producer (mirrors Rust's tokio::broadcast Lagged
// semantics). The primary `for await (item of handle)` iterator does
// NOT consume from this — it yields directly so the original API
// shape is unchanged.
const subscribers = new Set();
let driverDone = false;
function broadcastItem(v) {
for (const sub of subscribers) sub.push(v);
}
function broadcastClose() {
driverDone = true;
for (const sub of subscribers) sub.close();
}
// Seed: resume from checkpoint or from start URLs.
if (opts.resume) {
itemsEmitted = opts.resume.items_emitted || 0;
for (const s of opts.resume.seen || []) seen.add(s);
for (const cr of opts.resume.pending || []) {
pending.push({ url: cr.url, method: cr.method || 'GET', sessionId: cr.session_id });
}
} else {
for (const u of spider.startUrls) {
const r = { url: u, method: 'GET' };
if (seen.has(reqKey(r))) continue;
seen.add(reqKey(r));
pending.push(r);
}
}
function snapshot() {
return {
pending: pending.map((r) => ({ url: r.url, method: r.method, session_id: r.sessionId })),
seen: Array.from(seen),
items_emitted: itemsEmitted,
};
}
async function* iterate() {
while (pending.length > 0) {
if (signal && signal.aborted) { paused = true; break; }
if (spider.maxItems && itemsEmitted >= spider.maxItems) { paused = true; break; }
const req = pending.shift();
// Robots gate.
if (spider.robotsTxtObey) {
const host = hostOf(req.url);
const body = robotsCache.get(host);
if (body) {
const path = (() => { try { const u = new URL(req.url); return u.pathname + u.search; } catch { return '/'; } })();
const { allowed, crawlDelayMs } = evaluateRobots(body, spider.userAgent, path);
if (!allowed) continue;
if (crawlDelayMs > 0) {
const last = lastFetchPerHost.get(host);
const delta = last ? Date.now() - last : Infinity;
const wait = Math.max(0, crawlDelayMs - delta);
await sleep(wait);
}
}
}
// Per-domain throttle.
if (spider.downloadDelayMs > 0) {
const host = hostOf(req.url);
const last = lastFetchPerHost.get(host);
if (last !== undefined) {
const delta = Date.now() - last;
await sleep(Math.max(0, spider.downloadDelayMs - delta));
}
lastFetchPerHost.set(host, Date.now());
} else {
lastFetchPerHost.set(hostOf(req.url), Date.now());
}
let resp;
try {
resp = await fetcher({ ...req, userAgent: spider.userAgent });
} catch (_) {
continue;
}
const yields = spider.parse(resp);
const asyncIter = yields && typeof yields[Symbol.asyncIterator] === 'function'
? yields
: (yields && typeof yields[Symbol.iterator] === 'function' ? yields : [yields]);
for await (const y of asyncIter) {
if (y == null) continue;
if (y instanceof Request) {
const key = reqKey(y);
if (!seen.has(key)) {
seen.add(key);
pending.push({ url: y.url, method: y.method, sessionId: y.sessionId });
}
} else {
itemsEmitted += 1;
broadcastItem(y);
yield y;
}
}
}
}
const inner = iterate();
// Wrap the inner generator so we can close stream() subscribers when
// the primary iterator drains (or the caller breaks out of the loop).
async function* outer() {
try {
for await (const item of inner) yield item;
} finally {
broadcastClose();
}
}
const handle = outer();
handle.checkpoint = snapshot;
handle.isPaused = () => paused;
handle.stream = (sopts = {}) => createStream(subscribers, sopts.bufferSize || 1024, () => driverDone);
return handle;
}
function createStream(subscribers, bufferSize, isDone) {
// Bounded queue + wake-on-push. When the buffer is full we drop the
// oldest item — matches the tokio broadcast Lagged path (the bus
// stays alive; the slow consumer just loses history).
const queue = [];
let waiter = null;
let closed = false;
const sub = {
push(v) {
if (closed) return;
if (queue.length >= bufferSize) queue.shift();
queue.push(v);
if (waiter) {
const w = waiter;
waiter = null;
w();
}
},
close() {
closed = true;
if (waiter) {
const w = waiter;
waiter = null;
w();
}
},
};
subscribers.add(sub);
const it = (async function* () {
try {
while (true) {
if (queue.length > 0) {
yield queue.shift();
continue;
}
if (closed || isDone()) return;
await new Promise((r) => (waiter = r));
}
} finally {
subscribers.delete(sub);
}
})();
return it;
}
module.exports = {

@@ -553,2 +915,3 @@ crawl,

runJson,
paginatePages,
ensureInstalled,

@@ -560,2 +923,5 @@ binaryPath,

HOOK_EVENTS,
Request,
defineSpider,
runSpider,
version: SDK_VERSION,

@@ -562,0 +928,0 @@ };

+286
-9
sdk/index.d.ts

@@ -35,8 +35,37 @@ // Type surface for the crawlex Node SDK.

| 'vendor.telemetry_observed'
| 'tech.fingerprint_detected';
| 'tech.fingerprint_detected'
| 'item.scraped';
/**
* Canonical per-URL lifecycle status (slice 1). Mirrors
* `crawlex::Status` on the Rust side. Written to persisted crawl
* status rows and shipped on `BaseEnvelope.status`.
*/
export type UrlStatus =
| 'queued'
| 'completed'
| 'disallowed'
| 'skipped'
| 'errored'
| 'cancelled';
/**
* Canonical per-job terminal label (slice 1). Mirrors
* `crawlex::TerminalReason`. Written to persisted crawl telemetry.
*/
export type TerminalReason =
| 'completed'
| 'errored'
| 'cancelled_due_to_timeout'
| 'cancelled_due_to_limits'
| 'cancelled_by_user';
/** Outer envelope — every NDJSON line decodes into this shape. */
export interface BaseEnvelope<E extends EventKind = EventKind, D = unknown> {
/** Wire schema version. Currently `1`. */
v: 1;
/**
* Wire schema version. Currently `3` — bumped from `2` in slice 18
* when `item.scraped` joined the event taxonomy. Older consumers that
* only read `event`/`why`/`data` stay compatible.
*/
v: 3;
/** ISO-8601 UTC timestamp with millisecond precision. */

@@ -58,2 +87,7 @@ ts: string;

why?: string;
/**
* Canonical per-URL status (slice 1). Present on events that carry a
* per-URL lifecycle transition; optional everywhere else.
*/
status?: UrlStatus;
/** Event-specific payload. Shape varies per `event`. */

@@ -63,2 +97,62 @@ data: D;

// ─── pages list (SDK results endpoint) ─────────────────────────────────
/**
* Row returned by `crawlex pages list --json` (slice 1). The SDK
* results endpoint — call via [`runJson`] with `['pages', 'list',
* '--storage-path', path, '--status', 'completed']` to pull persisted
* rows filtered by canonical status.
*
* `crawl_status` is `null` on legacy rows written before the column
* existed; new writes populate it.
*/
export interface PageStatusRow {
url: string;
final_url: string;
/** Upstream HTTP status code (or `0` if the fetch never produced one). */
http_status: number;
crawl_status: UrlStatus | null;
}
/**
* Opaque pagination token from the SDK results read path (slice 8).
* Treated as an unstructured string by consumers — the encoding is a
* URL-safe base64 of a versioned struct and may change across crawlex
* releases. Pass it back verbatim on the next `pages list` call.
*/
export type PageCursor = string;
/**
* Wire shape of `crawlex pages list --json` (slice 8). When `limit > 0`,
* `next_cursor` is present iff additional rows match the filter. With
* `limit = 0` the entire result set ships in `rows` and `next_cursor`
* is absent.
*/
export interface PageListResponse {
rows: PageStatusRow[];
next_cursor?: PageCursor;
}
/** Options for [`paginatePages`]. */
export interface PaginatePagesOptions {
storagePath: string;
/** Optional canonical status filter (mirrors `--status`). */
status?: UrlStatus;
/** Rows per page. Defaults to `100`. Must be `> 0`. */
pageSize?: number;
/** Override the resolved binary path. */
bin?: string;
/** Extra env vars for the child process. */
env?: Record<string, string>;
}
/**
* Stream every persisted `pages` row matching `status`. Wraps
* `crawlex pages list` with cursor pagination so callers iterate
* without seeing the cursor token. Yields `PageStatusRow` values.
*/
export function paginatePages(
opts: PaginatePagesOptions
): AsyncIterableIterator<PageStatusRow>;
// ─── Typed payloads ────────────────────────────────────────────────────

@@ -126,3 +220,3 @@

* per-fetch network breakdown so a stream consumer can act on timings
* without round-tripping through the SQLite `page_metrics` table.
* without round-tripping through persisted page metrics.
*/

@@ -132,2 +226,6 @@ export interface FetchCompletedData {

status: number;
/** Which path served this URL: `"impersonate"` (HTTP spoof client) or
* `"fallback"` (external fallback_fetch command). The render path
* emits a separate `render.completed` event with `path: "render"`. */
path?: 'impersonate' | 'fallback';
bytes?: number;

@@ -168,2 +266,6 @@ body_truncated: boolean;

status: number;
/** Always `"render"` — the literal lets a stream consumer
* disambiguate against `fetch.completed.data.path` without
* inspecting the envelope's `event` field. */
path?: 'render';
manifest: boolean;

@@ -205,4 +307,4 @@ service_workers: number;

* (e.g. `artifacts/<session>/<stem>.png`).
* - SQLite backend: `cas:<sha256>` URI pointing at the
* content-addressed blob store (`<dbfile>.blobs/<shard>/<sha256>`).
* - RedDB/file-backed artifact stores may return stable `cas:<sha256>`
* style URIs for content-addressed payloads.
* - Memory backend / non-persisting sinks: omitted.

@@ -245,2 +347,15 @@ */

/**
* Mirrors `src/events/envelope.rs::ItemScrapedData` (slice 18). Emitted
* once per item yielded by a spider's `parse` invocation. Stream
* consumers built via `runSpider(...).stream()` see the `payload` shape;
* SDKs reading the raw event bus get the full descriptor including
* `spider_id` and the optional stable `identifier`.
*/
export interface ItemScrapedData {
spider_id: string;
identifier?: string;
payload: Record<string, unknown>;
}
export interface TechFingerprintDetectedData {

@@ -284,3 +399,4 @@ host: string;

| BaseEnvelope<'vendor.telemetry_observed', VendorTelemetryObservedData>
| BaseEnvelope<'tech.fingerprint_detected', TechFingerprintDetectedData>;
| BaseEnvelope<'tech.fingerprint_detected', TechFingerprintDetectedData>
| BaseEnvelope<'item.scraped', ItemScrapedData>;

@@ -347,5 +463,5 @@ /**

// ─── Storage / queue ─────────────────────────────────────────────
storage?: 'memory' | 'filesystem' | 'sqlite' | string;
storage?: 'memory' | 'filesystem' | 'reddb' | string;
storagePath?: string;
queue?: 'memory' | 'sqlite' | 'redis' | string;
queue?: 'reddb' | 'memory' | 'redis' | string;
queuePath?: string;

@@ -564,1 +680,162 @@ queueRedisUrl?: string;

export const version: string;
// ─── Selector engine (slices 9 & 11) ──────────────────────────────────
//
// Forward-declared types for the parser/selector surface. The runtime
// binding lands in a future SDK release (same deferral pattern as the
// streaming `paginate` helper above). Until then the type names are
// exported so application code can be written against the planned API.
/** CSS / XPath flavour for [`ElementHandle.generateSelector`]. */
export type SelectorKind = 'css' | 'xpath';
/**
* Handle to an element inside a parsed tree. Mirrors the rust
* `ElementHandle` surface — `css` / `xpath` queries scoped to this
* subtree, attribute / text access, navigation, and auto-selector
* generation (slice 11).
*/
export interface ElementHandle {
readonly tag: string;
attr(name: string): string | undefined;
text(): string;
html(): string;
innerHtml(): string;
parent(): ElementHandle | undefined;
children(): ElementHandle[];
siblings(): ElementHandle[];
css(selector: string): ElementHandle[];
xpath(expr: string): ElementHandle[];
/**
* Produce a selector string that uniquely identifies this element in
* its source tree. Prefers stable anchors (`id`, `data-testid`,
* ARIA attributes, semantic tags) over positional fallbacks
* (`:nth-of-type` for CSS, `[N]` for XPath).
*/
generateSelector(opts: { kind: SelectorKind }): string;
/**
* Find other elements in the same tree whose similarity score against
* this element meets `threshold` (default 0.2). The anchor itself is
* excluded. Results are sorted by descending score, so callers can
* `.slice(0, n)` for top-N matches. Pure in-tree scan — does not
* touch the adaptive store.
*/
findSimilar(opts?: { threshold?: number }): ElementHandle[];
}
/** Engine backend a session is bound to. */
export type BackendKind = 'http' | 'render' | 'stealth';
/** Options accepted by the [`Request`] constructor. */
export interface RequestOptions {
/** HTTP method. Defaults to `GET`. */
method?: string;
/**
* Optional session id. When supplied, the request runs against the
* backend + cookie jar registered for that id via `SessionManager`.
* Unknown ids log a warning and fall back to the default backend.
*/
sessionId?: string;
}
/**
* Recipe-facing request descriptor. Slice 16 plumbs `sessionId` so a
* recipe can pin successive fetches to an isolated engine state.
*/
export declare class Request {
constructor(url: string, opts?: RequestOptions);
readonly url: string;
readonly method: string;
readonly sessionId?: string;
}
// ─── Spider DSL (slice 17) ────────────────────────────────────────────
/** Response shape passed into `parse` by the JS spider runner. */
export interface SpiderResponse {
request: { url: string; method: string; sessionId?: string };
finalUrl: string;
status: number;
headers: Record<string, string | string[] | undefined>;
body: Buffer;
}
/** What `parse` yields. Items are arbitrary plain objects; new requests
* must be `Request` instances so the runner can dedupe by method+URL. */
export type ParseYield = Request | Record<string, unknown> | null | undefined;
export interface SpiderSpec {
/** Seed URLs. Required, must be non-empty. */
startUrls: string[];
/** Sync/async generator (or any iterable) that yields items and `Request`s. */
parse: (resp: SpiderResponse) =>
| Iterable<ParseYield>
| AsyncIterable<ParseYield>
| ParseYield;
/** Per-host minimum gap between fetches, in milliseconds. */
downloadDelayMs?: number;
/** Honour `Disallow` + `Crawl-delay` against `opts.robotsCache`. */
robotsTxtObey?: boolean;
/** UA string used in robots evaluation and the default fetcher. */
userAgent?: string;
/** Stop after N items emitted. */
maxItems?: number;
}
export interface SpiderDef extends Required<Omit<SpiderSpec, 'maxItems' | 'userAgent' | 'downloadDelayMs' | 'robotsTxtObey'>> {
startUrls: string[];
parse: SpiderSpec['parse'];
downloadDelayMs: number;
robotsTxtObey: boolean;
userAgent: string;
maxItems: number | null;
}
/** Persistable runner state. Field naming mirrors the Rust `Checkpoint`
* struct (snake_case) so a JS-paused run can be resumed in Rust. */
export interface SpiderCheckpoint {
pending: Array<{ url: string; method: string; session_id?: string }>;
seen: string[];
items_emitted: number;
}
export interface RunSpiderOptions {
/** Override the default node:http(s) fetcher. */
fetcher?: (req: { url: string; method: string; sessionId?: string; userAgent?: string }) =>
Promise<SpiderResponse>;
/** Map<host, robots.txt body>. Required when `robotsTxtObey` is true. */
robotsCache?: Map<string, string>;
/** Abort the run (Ctrl-C). Returned handle's `checkpoint()` captures
* the frontier at the pause point. */
signal?: AbortSignal;
/** Resume from a previously captured [`SpiderCheckpoint`]. */
resume?: SpiderCheckpoint;
}
/** Async iterable of items yielded by `parse`. Call `checkpoint()` after
* the loop terminates to snapshot the frontier for a resumable pause. */
export interface SpiderHandle extends AsyncIterableIterator<Record<string, unknown>> {
checkpoint(): SpiderCheckpoint;
isPaused(): boolean;
/**
* Independent broadcast subscriber over the items this spider yields.
* Calls return a fresh `AsyncIterable<Item>` each invocation — multiple
* consumers can `stream()` the same run in parallel. Backpressure: a
* slow subscriber sees the oldest items dropped from its bounded ring
* buffer (`bufferSize`, default 1024) rather than blocking the
* producer or crashing the bus. Mirrors the Rust
* `SpiderRunner::stream()` contract.
*
* Items only flow once the primary handle is iterated (or once the
* driver task has started), so consumers should typically subscribe
* before the first `await handle.next()` if they want a guaranteed
* count.
*/
stream(opts?: { bufferSize?: number }): AsyncIterableIterator<Record<string, unknown>>;
}
/** Validate + freeze a spider spec. */
export function defineSpider(spec: SpiderSpec): SpiderDef;
/** Drive a defined spider. */
export function runSpider(spider: SpiderDef, opts?: RunSpiderOptions): SpiderHandle;
+1
-1
sdk/version.js
// Auto-generated by scripts/sync-version.js. Do not edit by hand.
// Single source of truth: package.json.version.
'use strict';
module.exports = "1.0.4";
module.exports = "1.0.6-next.121";