crawlex - npm Package Compare versions

Comparing version

1.0.4

1.0.6

+345

sdk/shell.js

		'use strict';

		// crawlex shell (slice 23) — Node REPL with crawlex SDK preloaded.
		//
		// Mirrors the Rust shell (slice 22) helpers: `fetch`, `css`, `xpath`,
		// `findByText`, `findByRegex`, `save`. Implementation is intentionally
		// minimal — pure Node, no extra deps:
		// * fetch: node:http(s) GET
		// * selectors: regex-based subset (tag / tag#id / tag.class, //tag)
		// * adaptive save: JSON file under $XDG_DATA_HOME/crawlex
		//
		// Two layers (mirrors the Rust shell):
		// * `createShellApi` is the test surface — returns the helper object
		// and shared state without spinning up a REPL. Tests inject a stub
		// fetcher and assert on the state mutations.
		// * `startRepl` wires the API into `node:repl` with history.

		const repl = require('node:repl');
		const path = require('node:path');
		const fs = require('node:fs');
		const os = require('node:os');
		const http = require('node:http');
		const https = require('node:https');
		const { URL } = require('node:url');

		function xdgDataHome() {
		return process.env.XDG_DATA_HOME \|\| path.join(os.homedir(), '.local', 'share');
		}

		function defaultHistoryPath() {
		return path.join(xdgDataHome(), 'crawlex', 'shell_history_node');
		}

		function defaultStorePath() {
		return path.join(xdgDataHome(), 'crawlex', 'adaptive_store.json');
		}

		function httpFetch(url) {
		const u = new URL(url);
		const mod = u.protocol === 'https:' ? https : http;
		return new Promise((resolve, reject) => {
		const req = mod.request(
		{
		method: 'GET',
		hostname: u.hostname,
		port: u.port \|\| (u.protocol === 'https:' ? 443 : 80),
		path: u.pathname + u.search,
		headers: { 'user-agent': 'crawlex-shell' },
		},
		(res) => {
		const chunks = [];
		res.on('data', (c) => chunks.push(c));
		res.on('end', () => {
		resolve({
		finalUrl: url,
		status: res.statusCode \|\| 0,
		headers: res.headers,
		body: Buffer.concat(chunks).toString('utf8'),
		});
		});
		},
		);
		req.on('error', reject);
		req.end();
		});
		}

		function parseSimpleSelector(sel) {
		// Supports a minimal subset: tag, #id, .class, tag#id, tag.class.
		// Anything richer (descendant combinators, attribute selectors) is
		// out-of-scope for the JS shell — the Rust shell remains the heavy
		// engine.
		const m = /^([a-zA-Z][a-zA-Z0-9-])?(?:#([a-zA-Z0-9_-]+))?(?:\.([a-zA-Z0-9_-]+))?$/.exec(
		sel.trim(),
		);
		if (!m \|\| (!m[1] && !m[2] && !m[3])) return null;
		return { tag: m[1] \|\| '*', id: m[2] \|\| null, class: m[3] \|\| null };
		}

		function parseAttrs(s) {
		const out = {};
		const re = /\b([a-zA-Z][a-zA-Z0-9-])\s=\s(?:"([^"])"\|'([^']*)'\|([^\s>]+))/g;
		let m;
		while ((m = re.exec(s))) {
		out[m[1]] = m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3] : m[4];
		}
		return out;
		}

		const VOID_TAGS = new Set([
		'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
		'link', 'meta', 'param', 'source', 'track', 'wbr',
		]);

		// Walk every element in the document, including nested ones, yielding
		// `{ tag, attrs, inner, outer, index }`. The regex iterator approach
		// (matchAll on `<x>...</x>`) only catches top-level matches because the
		// engine advances past each match. Here we re-scan from every `<tag>`
		// open and resolve the matching `</tag>` with a depth counter so nested
		// elements with the same tag name (e.g. `<div>` inside `<div>`) close
		// correctly. O(n*k) for k opens — fine for shell-sized pages.
		function* elementIter(html) {
		const openRe = /<([a-zA-Z][a-zA-Z0-9-])\b([^>])>/g;
		let m;
		while ((m = openRe.exec(html))) {
		const tag = m[1];
		const attrs = m[2];
		if (VOID_TAGS.has(tag.toLowerCase())) continue;
		if (/\/\s*$/.test(attrs)) continue; // self-closing like <br/>
		const startInner = m.index + m[0].length;
		const close = findMatchingClose(html, tag, startInner);
		if (close < 0) continue;
		yield {
		0: html.slice(m.index, close + tag.length + 3),
		1: tag,
		2: attrs,
		3: html.slice(startInner, close),
		index: m.index,
		};
		}
		}

		function findMatchingClose(html, tag, from) {
		const openRe = new RegExp(`<${tag}\\b[^>]*>`, 'gi');
		const closeRe = new RegExp(`</${tag}\\s*>`, 'gi');
		let depth = 1;
		let pos = from;
		while (depth > 0) {
		openRe.lastIndex = pos;
		closeRe.lastIndex = pos;
		const o = openRe.exec(html);
		const c = closeRe.exec(html);
		if (!c) return -1;
		if (o && !/\/\s*$/.test(o[0].slice(1, -1)) && o.index < c.index) {
		depth += 1;
		pos = o.index + o[0].length;
		} else {
		depth -= 1;
		if (depth === 0) return c.index;
		pos = c.index + c[0].length;
		}
		}
		return -1;
		}

		function textOf(html) {
		return html
		.replace(/<script[\s\S]*?<\/script>/gi, ' ')
		.replace(/<style[\s\S]*?<\/style>/gi, ' ')
		.replace(/<[^>]+>/g, ' ')
		.replace(/ /g, ' ')
		.replace(/\s+/g, ' ')
		.trim();
		}

		function cssQuery(html, selector) {
		const parsed = parseSimpleSelector(selector);
		if (!parsed) {
		throw new Error(
		`unsupported selector '${selector}'. JS shell supports tag, #id, .class, tag#id, tag.class.`,
		);
		}
		const out = [];
		for (const m of elementIter(html)) {
		const tag = m[1];
		if (parsed.tag !== '*' && tag.toLowerCase() !== parsed.tag.toLowerCase()) continue;
		const attrs = parseAttrs(m[2]);
		if (parsed.id && attrs.id !== parsed.id) continue;
		if (parsed.class) {
		const classes = (attrs.class \|\| '').split(/\s+/);
		if (!classes.includes(parsed.class)) continue;
		}
		out.push({ tag, attrs, inner: m[3], outer: m[0], text: textOf(m[3]) });
		}
		return out;
		}

		function xpathQuery(html, expr) {
		// Supports `//tag` only. Anything richer goes to the Rust shell.
		const m = /^\/\/([a-zA-Z][a-zA-Z0-9-]*)$/.exec(expr.trim());
		if (!m) {
		throw new Error(`unsupported xpath '${expr}'. JS shell supports //tag only.`);
		}
		return cssQuery(html, m[1]);
		}

		function findByText(html, needle) {
		if (typeof needle !== 'string' \|\| needle.length === 0) {
		throw new TypeError('findByText: needle must be a non-empty string');
		}
		const out = [];
		for (const m of elementIter(html)) {
		const text = textOf(m[3]);
		if (text.includes(needle)) {
		out.push({ tag: m[1], attrs: parseAttrs(m[2]), inner: m[3], outer: m[0], text });
		}
		}
		return out;
		}

		function findByRegex(html, pattern) {
		const re = pattern instanceof RegExp ? pattern : new RegExp(pattern);
		const out = [];
		for (const m of elementIter(html)) {
		const text = textOf(m[3]);
		if (re.test(text)) {
		out.push({ tag: m[1], attrs: parseAttrs(m[2]), inner: m[3], outer: m[0], text });
		}
		}
		return out;
		}

		class Page {
		constructor(state, resp) {
		this._state = state;
		this.url = resp.finalUrl;
		this.status = resp.status;
		this.headers = resp.headers;
		this.body = resp.body;
		}
		_record(result, query) {
		this._state.lastSelection = result[0] \|\| null;
		this._state.lastSelectionQuery = query;
		this._state.lastSelectionUrl = this.url;
		return result;
		}
		css(selector) {
		return this._record(cssQuery(this.body, selector), `css:${selector}`);
		}
		xpath(expr) {
		return this._record(xpathQuery(this.body, expr), `xpath:${expr}`);
		}
		findByText(needle) {
		return this._record(findByText(this.body, needle), `text:${needle}`);
		}
		findByRegex(pattern) {
		const label = pattern instanceof RegExp ? pattern.source : String(pattern);
		return this._record(findByRegex(this.body, pattern), `regex:${label}`);
		}
		save(identifier) {
		if (typeof identifier !== 'string' \|\| !identifier) {
		throw new TypeError('save: identifier must be a non-empty string');
		}
		if (!this._state.lastSelection) {
		throw new Error('save: no element selected. Run css/xpath/findByText first.');
		}
		const host = new URL(this.url).host;
		const storePath = this._state.storePath;
		let store = {};
		try {
		store = JSON.parse(fs.readFileSync(storePath, 'utf8'));
		} catch (_) {
		store = {};
		}
		const entry = {
		query: this._state.lastSelectionQuery,
		tag: this._state.lastSelection.tag,
		attrs: this._state.lastSelection.attrs,
		saved_at: new Date().toISOString(),
		};
		store[host] = store[host] \|\| {};
		store[host][identifier] = entry;
		fs.mkdirSync(path.dirname(storePath), { recursive: true });
		fs.writeFileSync(storePath, JSON.stringify(store, null, 2));
		return entry;
		}
		}

		function createShellApi(opts = {}) {
		const fetcher = opts.fetcher \|\| httpFetch;
		const state = {
		last: null,
		lastSelection: null,
		lastSelectionQuery: null,
		lastSelectionUrl: null,
		storePath: opts.storePath \|\| defaultStorePath(),
		};
		const crawlex = {
		async fetch(url) {
		if (typeof url !== 'string' \|\| !url) {
		throw new TypeError('crawlex.fetch: url must be a non-empty string');
		}
		const resp = await fetcher(url);
		const page = new Page(state, resp);
		state.last = page;
		return page;
		},
		css(sel) { return requireLast(state).css(sel); },
		xpath(e) { return requireLast(state).xpath(e); },
		findByText(t) { return requireLast(state).findByText(t); },
		findByRegex(p) { return requireLast(state).findByRegex(p); },
		save(id) { return requireLast(state).save(id); },
		get last() { return state.last; },
		};
		return { crawlex, state };
		}

		function requireLast(state) {
		if (!state.last) {
		throw new Error('no page fetched yet — call crawlex.fetch(url) first.');
		}
		return state.last;
		}

		function startRepl(opts = {}) {
		const stdin = opts.stdin \|\| process.stdin;
		const stdout = opts.stdout \|\| process.stdout;
		const historyPath = opts.historyPath \|\| defaultHistoryPath();
		const { crawlex } = createShellApi({ storePath: opts.storePath });
		stdout.write(`crawlex shell — node ${process.version}\n`);
		stdout.write(
		`crawlex globals: fetch(url), css(sel), xpath(expr), findByText(s), findByRegex(re), save(id), last\n`,
		);
		stdout.write(`type .exit to quit.\n`);
		const server = repl.start({
		prompt: 'crawlex> ',
		input: stdin,
		output: stdout,
		useColors: stdout.isTTY === true,
		});
		server.context.crawlex = crawlex;
		Object.defineProperty(server.context, 'last', {
		get: () => crawlex.last,
		configurable: true,
		});
		try {
		fs.mkdirSync(path.dirname(historyPath), { recursive: true });
		server.setupHistory(historyPath, () => {});
		} catch (_) {
		// History is best-effort — non-fatal if the data dir is unwritable.
		}
		return server;
		}

		module.exports = {
		createShellApi,
		startRepl,
		Page,
		cssQuery,
		xpathQuery,
		findByText,
		findByRegex,
		defaultHistoryPath,
		defaultStorePath,
		};

+2

-1

package.json

		{
		"name": "crawlex",
		"version": "1.0.4",
		"version": "1.0.6",
		"description": "Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue",
		@@ -21,2 +21,3 @@ "type": "commonjs",
		"sdk/crawlex-sdk.js",
		"sdk/shell.js",
		"sdk/postinstall.js",
		@@ -23,0 +24,0 @@ "sdk/index.d.ts",

+66

-21

README.md

		@@ -38,3 +38,3 @@ <div align="center">
		<tr><td>📦 <strong>Catalog</strong></td><td>30 Chrome stable × 30 Chromium × 20 Firefox × Edge × Safari fingerprints. Era-fallback resolution: ask for <code>chrome-149-linux</code>, get the closest captured profile</td></tr>
		<tr><td>🛠️ <strong>Worker scope</strong></td><td>Same shim auto-attached to dedicated / shared / service workers via CDP <code>Target.setAutoAttach</code> — Camoufox port</td></tr>
		<tr><td>🛠️ <strong>Worker scope</strong></td><td>Same shim auto-attached to dedicated / shared / service workers via CDP <code>Target.setAutoAttach</code></td></tr>
		</table>
		@@ -63,2 +63,13 @@

		## 🆕 Last 24h highlights

		- `1.0.6` release line focuses on canonical URL identity, lossless frontier admission and cache/queue alias deduplication.
		- JS/TS hooks now run through the SDK bridge, so `defineHooks()` can drive the same lifecycle decisions as embedded Rust hooks.
		- NDJSON events now carry richer artifacts, Web Vitals, per-fetch timings, crawl-attempt telemetry and crawl-resolution summaries.
		- The supported release artifact is the full `crawlex` binary with RedDB persistence enabled.
		- Large crawl efficiency grew: cache validation, prefetch discovery mode and best-first URL scoring are now available from CLI/config.
		- Render fallback grew: external CDP connection, GPU posture control, Shadow DOM flattening, overlay cleanup and last-resort fallback fetch are configurable.

		---

		## 🏃 Quickstart
		@@ -197,6 +208,6 @@
		crtsh: true, // certificate-transparency seeding
		storage: 'sqlite',
		storagePath: './crawl.db',
		queue: 'sqlite',
		queuePath: './crawl.db',
		storage: 'reddb',
		storagePath: 'file://./state/crawl.rdb',
		queue: 'reddb',
		queuePath: 'file://./state/frontier.rdb',
		proxies: ['http://user:pass@proxy1:8080', 'http://user:pass@proxy2:8080'],
		@@ -299,2 +310,21 @@ proxyStrategy: 'health-weighted',

		### 6. Large crawl: validate cache, prefetch links, score the frontier

		```bash
		crawlex pages run \
		--seed https://docs.example.com \
		--method auto \
		--queue reddb --queue-path file://./state/frontier.rdb \
		--storage reddb --storage-path file://./state/crawl.rdb \
		--cache-validate \
		--cache-max-age-secs 86400 \
		--prefetch \
		--best-first \
		--score-keyword docs \
		--score-keyword api \
		--emit ndjson
		```

		This mode is for discovery passes: reuse fresh cache rows, harvest links cheaply, and let higher-value URLs rise in the queue before expensive render passes.

		---
		@@ -328,2 +358,4 @@
		- 🧬 Asset-ref classification (JS / CSS / image / API / nav)
		- ⚡ Prefetch mode for fast discovery-only passes
		- 🎯 Best-first URL scoring with keyword bonuses
		- 🔓 TCP port scan (opt-in, network-active)
		@@ -335,2 +367,4 @@
		- 🔄 Policy decisions: keep / drop / retry / scope-demote / proxy-rotate / give-up
		- 🧱 Unified block classifier with attempt-level crawl stats
		- 🪂 Fallback fetch command for last-resort HTML retrieval
		- 🎯 4 captcha solver adapters: in-house reCAPTCHA v3, 2captcha, anticaptcha, VLM
		@@ -343,4 +377,8 @@
		- 🎯 Render pool — Chromium auto-fetch + isolated user-data dirs
		- 🔁 Persistent queue: in-memory / SQLite / Redis backends
		- 💾 Storage: filesystem / SQLite / memory — opt-in per concern (artifact, state, challenge, telemetry, intel)
		- 🔌 External CDP endpoint support for managed/browser-farm Chrome
		- 🌑 Shadow DOM flattening + overlay / consent-popup cleanup
		- 🖥️ GPU policy: compatibility mode or stealth-friendly GPU surfaces
		- 🔁 Queue: RedDB-native durable frontier by default; memory remains only for isolated smokes/tests
		- 💾 Storage: RedDB for crawl state, telemetry and intel; filesystem remains available for large artifacts
		- 🧠 Smart cache validation: `ETag`, `Last-Modified`, `<head>` fingerprint
		- 🔄 Proxy rotator — health checks + sticky sessions + per-host affinity
		@@ -354,5 +392,6 @@ - 📊 Web Vitals + per-fetch network breakdown (DNS / TCP / TLS / TTFB / download)
		- 📜 NDJSON event stream — versioned envelope (`v: 1`)
		- 🎬 19 event kinds covering full lifecycle
		- 🎬 21 event kinds covering full lifecycle
		- 🔬 Embedded `WebVitals` summary on `render.completed`
		- ⏱️ Per-request timings on `fetch.completed` (ALPN, cipher, TLS version)
		- 🧾 `crawl.attempted` / `crawl.resolved` telemetry for HTTP → render → fallback ladders
		- 📸 Artifact descriptors with on-disk path on the wire
		@@ -367,6 +406,7 @@ - 🪝 Hooks: 12 lifecycle points × 3 languages (Rust / JS / Lua)
		- 🔌 SDK `crawl()` async iterator
		- 🧩 SDK `defineHooks()` bridge for JS/TS lifecycle hooks
		- 📚 docsify docs site (GitHub Pages)
		- 🧪 386+ lib tests, 27 fpjs compliance, TLS catalog roundtrip suite
		- 🧪 820+ lib tests, fpjs compliance, TLS catalog roundtrip suite
		- 🔐 Optional Lua hooks (`mlua`)
		- 🪶 Two binaries: `crawlex` (full) + `crawlex-mini` (HTTP-only, no Chromium)
		- 🪶 Single supported release binary: `crawlex` with HTTP impersonation, Chromium rendering and RedDB persistence

		@@ -381,3 +421,3 @@ </td>

		Every run emits one JSON envelope per line on stdout. Versioned, stable, 19 kinds:
		Every run emits one JSON envelope per line on stdout. Versioned, stable, 21 kinds:

		@@ -388,2 +428,3 @@ ```jsonl
		{"v":1,"event":"fetch.completed","run_id":42,"url":"https://target.com/","data":{"final_url":"https://target.com/","status":200,"bytes":98234,"body_truncated":false,"dns_ms":12,"tcp_connect_ms":18,"tls_handshake_ms":24,"ttfb_ms":142,"download_ms":83,"total_ms":280,"alpn":"h2","tls_version":"TLSv1.3","cipher":"TLS_AES_128_GCM_SHA256"}}
		{"v":1,"event":"crawl.attempted","run_id":42,"url":"https://target.com/","data":{"crawl_id":42,"attempt_index":1,"engine":"http_spoof","status":403,"blocked":true,"block_reason":"Cloudflare challenge form"}}
		{"v":1,"event":"render.completed","run_id":42,"session_id":"sess_abc","url":"https://target.com/","data":{"final_url":"https://target.com/","status":200,"manifest":true,"service_workers":1,"is_spa":true,"vitals":{"ttfb_ms":142,"first_contentful_paint_ms":380.5,"largest_contentful_paint_ms":920.1,"cumulative_layout_shift":0.03,"total_blocking_time_ms":50.0,"dom_nodes":1842,"js_heap_used_bytes":12345678,"resource_count":45,"total_transfer_bytes":982341}}}
		@@ -393,2 +434,3 @@ {"v":1,"event":"artifact.saved","run_id":42,"url":"https://target.com/","data":{"kind":"screenshot.full_page","mime":"image/png","size":1234567,"sha256":"a1b2c3...","path":"artifacts/sess_abc/1714123456_screenshot_full_page_a1b2c3d4.png"}}
		{"v":1,"event":"decision.made","run_id":42,"url":"https://protected.com/","why":"render:js-challenge","data":{"decision":"retry","reason":{"code":"render:js-challenge"}}}
		{"v":1,"event":"crawl.resolved","run_id":42,"url":"https://target.com/","data":{"crawl_id":42,"attempts_count":2,"fallback_fetch_used":false,"resolved_by":"render","success":true}}
		{"v":1,"event":"run.completed","run_id":42}
		@@ -444,3 +486,6 @@ ```
		Q --> P[Policy Engine]
		P -->\|http\| F[ImpersonateClient<br/>BoringSSL + h2 patched]
		P --> C[Cache Validator<br/>ETag + Last-Modified + head fingerprint]
		C -->\|fresh\| ST[Storage<br/>5 traits]
		C -->\|stale\| F[ImpersonateClient<br/>BoringSSL + h2 patched]
		P -->\|http\| F
		P -->\|render\| R[RenderPool<br/>Chromium + stealth shim]
		@@ -450,5 +495,5 @@ F --> X[Extractor<br/>+ Asset Refs]
		X --> D[Discovery<br/>Pipeline]
		X --> ST[Storage<br/>5 traits]
		X --> ST
		D --> Q
		P --> EV[NDJSON Events<br/>19 kinds]
		P --> EV[NDJSON Events<br/>21 kinds]
		R --> H1[Rust Hooks]
		@@ -465,2 +510,3 @@ R --> H2[JS Bridge]
		- `antibot/` — vendor classifier + 4 captcha solver adapters
		- `cache_validator/` — cache freshness by HTTP validators and head fingerprints
		- `storage/` — 5 concern-oriented traits (artifact / state / challenge / telemetry / intel)
		@@ -480,3 +526,3 @@ - `events/` — NDJSON envelope + sink (stdout / null / memory)
		\| Async \| tokio multi-thread \|
		\| Storage \| rusqlite (SQLite WAL), DashMap (memory), filesystem layout \|
		\| Storage / Queue \| RedDB embedded/client storage plus RedDB Queue frontier; DashMap memory only for isolated tests \|
		\| Discovery \| hickory-resolver (DNS), reqwest (RDAP), texting_robots (robots.txt) \|
		@@ -486,5 +532,4 @@ \| Lua \| mlua 0.10 (optional, `lua-hooks` feature) \|

		Two binaries ship from one source tree:
		- `crawlex` — full build with HTTP impersonation + Chromium rendering + stealth shim + persistent queue
		- `crawlex-mini` — HTTP-only worker, no Chromium dependency, same CLI surface (browser-only flags return `Error::RenderDisabled`)
		Release binary:
		- `crawlex` — full build with HTTP impersonation + Chromium rendering + stealth shim + RedDB persistence

		@@ -501,4 +546,4 @@ ---
		\| Worker-scope stealth \| ✅ auto-attach \| ⚠️ manual \| ⚠️ manual \| ❌ \|
		\| HTTP-only path (no browser) \| ✅ `crawlex-mini` \| ❌ \| ❌ \| ✅ \|
		\| Persistent queue + resume \| ✅ SQLite/Redis \| ❌ external \| ❌ external \| ❌ \|
		\| HTTP-only path (no browser) \| ✅ full binary policy path \| ❌ \| ❌ \| ✅ \|
		\| RedDB telemetry + resume state \| ✅ native \| ❌ external \| ❌ external \| ❌ \|
		\| Discovery pipeline \| ✅ 17 stages \| ❌ \| ❌ \| ❌ \|
		@@ -530,3 +575,3 @@ \| Streaming NDJSON events \| ✅ versioned \| ❌ \| ❌ \| ❌ \|
		# Unit tests + offline shim compliance
		cargo test --lib # 386+ tests
		cargo test --lib # 820+ tests
		cargo test --test fpjs_compliance # 27 cases
		@@ -533,0 +578,0 @@ cargo test --test tls_catalog_coverage --test tls_catalog_roundtrip

+366

-0

sdk/crawlex-sdk.js

		@@ -387,2 +387,10 @@ #!/usr/bin/env node
		function runCli(argv) {
		// Slice 23 — `crawlex shell` is a Node REPL handled in JS, not a
		// passthrough to the rust binary. Match only the bare `shell`
		// subcommand; flags like `--help` after `shell` still want help text
		// from the native binary if it ever grows one.
		if (argv[0] === 'shell' && argv.length === 1) {
		require('./shell.js').startRepl();
		return;
		}
		const bin = binaryPath();
		@@ -548,2 +556,356 @@ if (!fs.existsSync(bin) && !process.env.CRAWLEX_FORCE_BINARY) {

		/**
		* Async iterator over `crawlex pages list` results (slice 8). Hides
		* the cursor token from the caller — internally re-invokes the binary
		* once per page using the prior response's `next_cursor` and
		* terminates when the server stops returning one.
		*/
		async function* paginatePages(opts) {
		if (!opts \|\| typeof opts.storagePath !== 'string' \|\| !opts.storagePath) {
		throw new TypeError('paginatePages: opts.storagePath is required');
		}
		const pageSize = opts.pageSize == null ? 100 : Number(opts.pageSize);
		if (!Number.isInteger(pageSize) \|\| pageSize <= 0) {
		throw new RangeError(`paginatePages: pageSize must be a positive integer (got ${opts.pageSize})`);
		}
		const runOpts = {};
		if (opts.bin) runOpts.bin = opts.bin;
		if (opts.env) runOpts.env = opts.env;

		let cursor = null;
		while (true) {
		const argv = ['pages', 'list', '--storage-path', opts.storagePath, '--limit', String(pageSize)];
		if (opts.status) argv.push('--status', opts.status);
		if (cursor) argv.push('--cursor', cursor);
		const resp = runJson(argv, runOpts);
		if (!resp \|\| !Array.isArray(resp.rows)) {
		throw new Error('paginatePages: unexpected response shape from `pages list`');
		}
		for (const row of resp.rows) yield row;
		if (!resp.next_cursor) return;
		cursor = resp.next_cursor;
		}
		}

		// --- v2 scraping framework: Request (slice 16) ------------------------
		// Pure JS placeholder. Runtime dispatch against the Rust SessionManager
		// lands once the engine bindings ship; the class exists today so recipes
		// can already author against the documented shape.
		class Request {
		constructor(url, opts = {}) {
		if (typeof url !== 'string' \|\| url.length === 0) {
		throw new TypeError('Request: url must be a non-empty string');
		}
		this.url = url;
		this.method = typeof opts.method === 'string' ? opts.method : 'GET';
		if (opts.sessionId !== undefined) {
		if (typeof opts.sessionId !== 'string' \|\| opts.sessionId.length === 0) {
		throw new TypeError('Request: sessionId must be a non-empty string');
		}
		this.sessionId = opts.sessionId;
		}
		}
		}

		// --- v2 scraping framework: defineSpider / runSpider (slice 17) ------
		// Pure JS spider runtime: the Rust `SpiderRunner` is the production
		// driver, but a self-contained JS implementation is shipped here so
		// recipes can run end-to-end against a Node http(s) fetcher today.
		// Both share the same Checkpoint wire shape (see `src/scraping/spider.rs`)
		// so a JS run can pause and resume in Rust (or vice versa) once the
		// engine binding lands.

		function defineSpider(spec = {}) {
		if (!spec \|\| typeof spec !== 'object') {
		throw new TypeError('defineSpider: spec must be an object');
		}
		if (!Array.isArray(spec.startUrls) \|\| spec.startUrls.length === 0) {
		throw new TypeError('defineSpider: startUrls must be a non-empty array');
		}
		if (typeof spec.parse !== 'function') {
		throw new TypeError('defineSpider: parse must be a function');
		}
		return {
		startUrls: spec.startUrls.slice(),
		parse: spec.parse,
		downloadDelayMs: Number(spec.downloadDelayMs) \|\| 0,
		robotsTxtObey: Boolean(spec.robotsTxtObey),
		userAgent: typeof spec.userAgent === 'string' ? spec.userAgent : 'crawlex',
		maxItems: Number.isInteger(spec.maxItems) && spec.maxItems > 0 ? spec.maxItems : null,
		};
		}

		function reqKey(r) {
		return `${r.method \|\| 'GET'} ${r.url}`;
		}

		function hostOf(url) {
		try { return new URL(url).host; } catch (_) { return ''; }
		}

		// Minimal robots.txt evaluator — Disallow + optional Crawl-delay only.
		// Honours UA-specific blocks falling back to `*`. Anything beyond what
		// slice 17 promises (no Allow precedence, no path globs beyond simple
		// prefix) is intentionally deferred to the Rust dispatcher.
		function evaluateRobots(body, userAgent, urlPath) {
		if (!body) return { allowed: true, crawlDelayMs: 0 };
		const uaLc = userAgent.toLowerCase();
		let currentUAs = [];
		let bestSpecificity = -1; // -1 none, 0 wildcard, 1 exact
		let allowed = true;
		let crawlDelayMs = 0;
		let activeBlock = false;
		for (const raw of body.split(/\r?\n/)) {
		const line = raw.replace(/#.*$/, '').trim();
		if (!line) {
		currentUAs = [];
		continue;
		}
		const m = line.match(/^([A-Za-z-]+)\s:\s(.*)$/);
		if (!m) continue;
		const key = m[1].toLowerCase();
		const val = m[2].trim();
		if (key === 'user-agent') {
		currentUAs.push(val.toLowerCase());
		const spec = currentUAs.includes(uaLc) ? 1 : currentUAs.includes('*') ? 0 : -1;
		activeBlock = spec >= bestSpecificity;
		if (spec > bestSpecificity) {
		bestSpecificity = spec;
		allowed = true;
		crawlDelayMs = 0;
		}
		continue;
		}
		if (!activeBlock) continue;
		if (key === 'disallow') {
		if (val && urlPath.startsWith(val)) allowed = false;
		} else if (key === 'crawl-delay') {
		const n = Number(val);
		if (Number.isFinite(n)) crawlDelayMs = Math.max(crawlDelayMs, Math.round(n * 1000));
		}
		}
		return { allowed, crawlDelayMs };
		}

		function sleep(ms) {
		return ms > 0 ? new Promise((r) => setTimeout(r, ms)) : Promise.resolve();
		}

		async function defaultFetcher(req) {
		const http = require('node:http');
		const https = require('node:https');
		const url = new URL(req.url);
		const mod = url.protocol === 'https:' ? https : http;
		return await new Promise((resolve, reject) => {
		const r = mod.request(
		{
		method: req.method \|\| 'GET',
		hostname: url.hostname,
		port: url.port \|\| (url.protocol === 'https:' ? 443 : 80),
		path: url.pathname + url.search,
		headers: { 'user-agent': req.userAgent \|\| 'crawlex' },
		},
		(res) => {
		const chunks = [];
		res.on('data', (c) => chunks.push(c));
		res.on('end', () => {
		resolve({
		request: req,
		finalUrl: req.url,
		status: res.statusCode \|\| 0,
		headers: res.headers,
		body: Buffer.concat(chunks),
		});
		});
		},
		);
		r.on('error', reject);
		r.end();
		});
		}

		/**
		* Drive a spider built by [`defineSpider`]. Async generator yielding
		* each emitted item. Honours `AbortSignal` (Ctrl-C path) and returns a
		* resumable checkpoint via `handle.checkpoint()` (call after `for await`
		* loop exits).
		*/
		function runSpider(spider, opts = {}) {
		const fetcher = opts.fetcher \|\| defaultFetcher;
		const robotsCache = opts.robotsCache \|\| new Map(); // host -> body
		const signal = opts.signal;

		const seen = new Set();
		const pending = [];
		const lastFetchPerHost = new Map();
		let itemsEmitted = 0;
		let paused = false;

		// Bounded broadcaster for `stream()` consumers. Each subscriber holds
		// its own ring buffer; overflow drops the oldest item rather than
		// blocking the producer (mirrors Rust's tokio::broadcast Lagged
		// semantics). The primary `for await (item of handle)` iterator does
		// NOT consume from this — it yields directly so the original API
		// shape is unchanged.
		const subscribers = new Set();
		let driverDone = false;

		function broadcastItem(v) {
		for (const sub of subscribers) sub.push(v);
		}
		function broadcastClose() {
		driverDone = true;
		for (const sub of subscribers) sub.close();
		}

		// Seed: resume from checkpoint or from start URLs.
		if (opts.resume) {
		itemsEmitted = opts.resume.items_emitted \|\| 0;
		for (const s of opts.resume.seen \|\| []) seen.add(s);
		for (const cr of opts.resume.pending \|\| []) {
		pending.push({ url: cr.url, method: cr.method \|\| 'GET', sessionId: cr.session_id });
		}
		} else {
		for (const u of spider.startUrls) {
		const r = { url: u, method: 'GET' };
		if (seen.has(reqKey(r))) continue;
		seen.add(reqKey(r));
		pending.push(r);
		}
		}

		function snapshot() {
		return {
		pending: pending.map((r) => ({ url: r.url, method: r.method, session_id: r.sessionId })),
		seen: Array.from(seen),
		items_emitted: itemsEmitted,
		};
		}

		async function* iterate() {
		while (pending.length > 0) {
		if (signal && signal.aborted) { paused = true; break; }
		if (spider.maxItems && itemsEmitted >= spider.maxItems) { paused = true; break; }
		const req = pending.shift();

		// Robots gate.
		if (spider.robotsTxtObey) {
		const host = hostOf(req.url);
		const body = robotsCache.get(host);
		if (body) {
		const path = (() => { try { const u = new URL(req.url); return u.pathname + u.search; } catch { return '/'; } })();
		const { allowed, crawlDelayMs } = evaluateRobots(body, spider.userAgent, path);
		if (!allowed) continue;
		if (crawlDelayMs > 0) {
		const last = lastFetchPerHost.get(host);
		const delta = last ? Date.now() - last : Infinity;
		const wait = Math.max(0, crawlDelayMs - delta);
		await sleep(wait);
		}
		}
		}

		// Per-domain throttle.
		if (spider.downloadDelayMs > 0) {
		const host = hostOf(req.url);
		const last = lastFetchPerHost.get(host);
		if (last !== undefined) {
		const delta = Date.now() - last;
		await sleep(Math.max(0, spider.downloadDelayMs - delta));
		}
		lastFetchPerHost.set(host, Date.now());
		} else {
		lastFetchPerHost.set(hostOf(req.url), Date.now());
		}

		let resp;
		try {
		resp = await fetcher({ ...req, userAgent: spider.userAgent });
		} catch (_) {
		continue;
		}

		const yields = spider.parse(resp);
		const asyncIter = yields && typeof yields[Symbol.asyncIterator] === 'function'
		? yields
		: (yields && typeof yields[Symbol.iterator] === 'function' ? yields : [yields]);
		for await (const y of asyncIter) {
		if (y == null) continue;
		if (y instanceof Request) {
		const key = reqKey(y);
		if (!seen.has(key)) {
		seen.add(key);
		pending.push({ url: y.url, method: y.method, sessionId: y.sessionId });
		}
		} else {
		itemsEmitted += 1;
		broadcastItem(y);
		yield y;
		}
		}
		}
		}

		const inner = iterate();
		// Wrap the inner generator so we can close stream() subscribers when
		// the primary iterator drains (or the caller breaks out of the loop).
		async function* outer() {
		try {
		for await (const item of inner) yield item;
		} finally {
		broadcastClose();
		}
		}
		const handle = outer();
		handle.checkpoint = snapshot;
		handle.isPaused = () => paused;
		handle.stream = (sopts = {}) => createStream(subscribers, sopts.bufferSize \|\| 1024, () => driverDone);
		return handle;
		}

		function createStream(subscribers, bufferSize, isDone) {
		// Bounded queue + wake-on-push. When the buffer is full we drop the
		// oldest item — matches the tokio broadcast Lagged path (the bus
		// stays alive; the slow consumer just loses history).
		const queue = [];
		let waiter = null;
		let closed = false;
		const sub = {
		push(v) {
		if (closed) return;
		if (queue.length >= bufferSize) queue.shift();
		queue.push(v);
		if (waiter) {
		const w = waiter;
		waiter = null;
		w();
		}
		},
		close() {
		closed = true;
		if (waiter) {
		const w = waiter;
		waiter = null;
		w();
		}
		},
		};
		subscribers.add(sub);
		const it = (async function* () {
		try {
		while (true) {
		if (queue.length > 0) {
		yield queue.shift();
		continue;
		}
		if (closed \|\| isDone()) return;
		await new Promise((r) => (waiter = r));
		}
		} finally {
		subscribers.delete(sub);
		}
		})();
		return it;
		}

		module.exports = {
		@@ -553,2 +915,3 @@ crawl,
		runJson,
		paginatePages,
		ensureInstalled,
		@@ -560,2 +923,5 @@ binaryPath,
		HOOK_EVENTS,
		Request,
		defineSpider,
		runSpider,
		version: SDK_VERSION,
		@@ -562,0 +928,0 @@ };

+286

-9

sdk/index.d.ts

		@@ -35,8 +35,37 @@ // Type surface for the crawlex Node SDK.
		\| 'vendor.telemetry_observed'
		\| 'tech.fingerprint_detected';
		\| 'tech.fingerprint_detected'
		\| 'item.scraped';

		/**
		* Canonical per-URL lifecycle status (slice 1). Mirrors
		* `crawlex::Status` on the Rust side. Written to persisted crawl
		* status rows and shipped on `BaseEnvelope.status`.
		*/
		export type UrlStatus =
		\| 'queued'
		\| 'completed'
		\| 'disallowed'
		\| 'skipped'
		\| 'errored'
		\| 'cancelled';

		/**
		* Canonical per-job terminal label (slice 1). Mirrors
		* `crawlex::TerminalReason`. Written to persisted crawl telemetry.
		*/
		export type TerminalReason =
		\| 'completed'
		\| 'errored'
		\| 'cancelled_due_to_timeout'
		\| 'cancelled_due_to_limits'
		\| 'cancelled_by_user';

		/** Outer envelope — every NDJSON line decodes into this shape. */
		export interface BaseEnvelope<E extends EventKind = EventKind, D = unknown> {
		/** Wire schema version. Currently `1`. */
		v: 1;
		/**
		* Wire schema version. Currently `3` — bumped from `2` in slice 18
		* when `item.scraped` joined the event taxonomy. Older consumers that
		* only read `event`/`why`/`data` stay compatible.
		*/
		v: 3;
		/** ISO-8601 UTC timestamp with millisecond precision. */
		@@ -58,2 +87,7 @@ ts: string;
		why?: string;
		/**
		* Canonical per-URL status (slice 1). Present on events that carry a
		* per-URL lifecycle transition; optional everywhere else.
		*/
		status?: UrlStatus;
		/** Event-specific payload. Shape varies per `event`. */
		@@ -63,2 +97,62 @@ data: D;

		// ─── pages list (SDK results endpoint) ─────────────────────────────────

		/**
		* Row returned by `crawlex pages list --json` (slice 1). The SDK
		* results endpoint — call via [`runJson`] with `['pages', 'list',
		* '--storage-path', path, '--status', 'completed']` to pull persisted
		* rows filtered by canonical status.
		*
		* `crawl_status` is `null` on legacy rows written before the column
		* existed; new writes populate it.
		*/
		export interface PageStatusRow {
		url: string;
		final_url: string;
		/** Upstream HTTP status code (or `0` if the fetch never produced one). */
		http_status: number;
		crawl_status: UrlStatus \| null;
		}

		/**
		* Opaque pagination token from the SDK results read path (slice 8).
		* Treated as an unstructured string by consumers — the encoding is a
		* URL-safe base64 of a versioned struct and may change across crawlex
		* releases. Pass it back verbatim on the next `pages list` call.
		*/
		export type PageCursor = string;

		/**
		* Wire shape of `crawlex pages list --json` (slice 8). When `limit > 0`,
		* `next_cursor` is present iff additional rows match the filter. With
		* `limit = 0` the entire result set ships in `rows` and `next_cursor`
		* is absent.
		*/
		export interface PageListResponse {
		rows: PageStatusRow[];
		next_cursor?: PageCursor;
		}

		/** Options for [`paginatePages`]. */
		export interface PaginatePagesOptions {
		storagePath: string;
		/** Optional canonical status filter (mirrors `--status`). */
		status?: UrlStatus;
		/** Rows per page. Defaults to `100`. Must be `> 0`. */
		pageSize?: number;
		/** Override the resolved binary path. */
		bin?: string;
		/** Extra env vars for the child process. */
		env?: Record<string, string>;
		}

		/**
		* Stream every persisted `pages` row matching `status`. Wraps
		* `crawlex pages list` with cursor pagination so callers iterate
		* without seeing the cursor token. Yields `PageStatusRow` values.
		*/
		export function paginatePages(
		opts: PaginatePagesOptions
		): AsyncIterableIterator<PageStatusRow>;

		// ─── Typed payloads ────────────────────────────────────────────────────
		@@ -126,3 +220,3 @@
		* per-fetch network breakdown so a stream consumer can act on timings
		* without round-tripping through the SQLite `page_metrics` table.
		* without round-tripping through persisted page metrics.
		*/
		@@ -132,2 +226,6 @@ export interface FetchCompletedData {
		status: number;
		/** Which path served this URL: `"impersonate"` (HTTP spoof client) or
		* `"fallback"` (external fallback_fetch command). The render path
		* emits a separate `render.completed` event with `path: "render"`. */
		path?: 'impersonate' \| 'fallback';
		bytes?: number;
		@@ -168,2 +266,6 @@ body_truncated: boolean;
		status: number;
		/** Always `"render"` — the literal lets a stream consumer
		* disambiguate against `fetch.completed.data.path` without
		* inspecting the envelope's `event` field. */
		path?: 'render';
		manifest: boolean;
		@@ -205,4 +307,4 @@ service_workers: number;
		* (e.g. `artifacts/<session>/<stem>.png`).
		* - SQLite backend: `cas:<sha256>` URI pointing at the
		* content-addressed blob store (`<dbfile>.blobs/<shard>/<sha256>`).
		* - RedDB/file-backed artifact stores may return stable `cas:<sha256>`
		* style URIs for content-addressed payloads.
		* - Memory backend / non-persisting sinks: omitted.
		@@ -245,2 +347,15 @@ */

		/**
		* Mirrors `src/events/envelope.rs::ItemScrapedData` (slice 18). Emitted
		* once per item yielded by a spider's `parse` invocation. Stream
		* consumers built via `runSpider(...).stream()` see the `payload` shape;
		* SDKs reading the raw event bus get the full descriptor including
		* `spider_id` and the optional stable `identifier`.
		*/
		export interface ItemScrapedData {
		spider_id: string;
		identifier?: string;
		payload: Record<string, unknown>;
		}

		export interface TechFingerprintDetectedData {
		@@ -284,3 +399,4 @@ host: string;
		\| BaseEnvelope<'vendor.telemetry_observed', VendorTelemetryObservedData>
		\| BaseEnvelope<'tech.fingerprint_detected', TechFingerprintDetectedData>;
		\| BaseEnvelope<'tech.fingerprint_detected', TechFingerprintDetectedData>
		\| BaseEnvelope<'item.scraped', ItemScrapedData>;

		@@ -347,5 +463,5 @@ /**
		// ─── Storage / queue ─────────────────────────────────────────────
		storage?: 'memory' \| 'filesystem' \| 'sqlite' \| string;
		storage?: 'memory' \| 'filesystem' \| 'reddb' \| string;
		storagePath?: string;
		queue?: 'memory' \| 'sqlite' \| 'redis' \| string;
		queue?: 'reddb' \| 'memory' \| 'redis' \| string;
		queuePath?: string;
		@@ -564,1 +680,162 @@ queueRedisUrl?: string;
		export const version: string;

		// ─── Selector engine (slices 9 & 11) ──────────────────────────────────
		//
		// Forward-declared types for the parser/selector surface. The runtime
		// binding lands in a future SDK release (same deferral pattern as the
		// streaming `paginate` helper above). Until then the type names are
		// exported so application code can be written against the planned API.

		/** CSS / XPath flavour for [`ElementHandle.generateSelector`]. */
		export type SelectorKind = 'css' \| 'xpath';

		/**
		* Handle to an element inside a parsed tree. Mirrors the rust
		* `ElementHandle` surface — `css` / `xpath` queries scoped to this
		* subtree, attribute / text access, navigation, and auto-selector
		* generation (slice 11).
		*/
		export interface ElementHandle {
		readonly tag: string;
		attr(name: string): string \| undefined;
		text(): string;
		html(): string;
		innerHtml(): string;
		parent(): ElementHandle \| undefined;
		children(): ElementHandle[];
		siblings(): ElementHandle[];
		css(selector: string): ElementHandle[];
		xpath(expr: string): ElementHandle[];
		/**
		* Produce a selector string that uniquely identifies this element in
		* its source tree. Prefers stable anchors (`id`, `data-testid`,
		* ARIA attributes, semantic tags) over positional fallbacks
		* (`:nth-of-type` for CSS, `[N]` for XPath).
		*/
		generateSelector(opts: { kind: SelectorKind }): string;
		/**
		* Find other elements in the same tree whose similarity score against
		* this element meets `threshold` (default 0.2). The anchor itself is
		* excluded. Results are sorted by descending score, so callers can
		* `.slice(0, n)` for top-N matches. Pure in-tree scan — does not
		* touch the adaptive store.
		*/
		findSimilar(opts?: { threshold?: number }): ElementHandle[];
		}

		/** Engine backend a session is bound to. */
		export type BackendKind = 'http' \| 'render' \| 'stealth';

		/** Options accepted by the [`Request`] constructor. */
		export interface RequestOptions {
		/** HTTP method. Defaults to `GET`. */
		method?: string;
		/**
		* Optional session id. When supplied, the request runs against the
		* backend + cookie jar registered for that id via `SessionManager`.
		* Unknown ids log a warning and fall back to the default backend.
		*/
		sessionId?: string;
		}

		/**
		* Recipe-facing request descriptor. Slice 16 plumbs `sessionId` so a
		* recipe can pin successive fetches to an isolated engine state.
		*/
		export declare class Request {
		constructor(url: string, opts?: RequestOptions);
		readonly url: string;
		readonly method: string;
		readonly sessionId?: string;
		}

		// ─── Spider DSL (slice 17) ────────────────────────────────────────────

		/** Response shape passed into `parse` by the JS spider runner. */
		export interface SpiderResponse {
		request: { url: string; method: string; sessionId?: string };
		finalUrl: string;
		status: number;
		headers: Record<string, string \| string[] \| undefined>;
		body: Buffer;
		}

		/** What `parse` yields. Items are arbitrary plain objects; new requests
		* must be `Request` instances so the runner can dedupe by method+URL. */
		export type ParseYield = Request \| Record<string, unknown> \| null \| undefined;

		export interface SpiderSpec {
		/** Seed URLs. Required, must be non-empty. */
		startUrls: string[];
		/** Sync/async generator (or any iterable) that yields items and `Request`s. */
		parse: (resp: SpiderResponse) =>
		\| Iterable<ParseYield>
		\| AsyncIterable<ParseYield>
		\| ParseYield;
		/** Per-host minimum gap between fetches, in milliseconds. */
		downloadDelayMs?: number;
		/** Honour `Disallow` + `Crawl-delay` against `opts.robotsCache`. */
		robotsTxtObey?: boolean;
		/** UA string used in robots evaluation and the default fetcher. */
		userAgent?: string;
		/** Stop after N items emitted. */
		maxItems?: number;
		}

		export interface SpiderDef extends Required<Omit<SpiderSpec, 'maxItems' \| 'userAgent' \| 'downloadDelayMs' \| 'robotsTxtObey'>> {
		startUrls: string[];
		parse: SpiderSpec['parse'];
		downloadDelayMs: number;
		robotsTxtObey: boolean;
		userAgent: string;
		maxItems: number \| null;
		}

		/** Persistable runner state. Field naming mirrors the Rust `Checkpoint`
		* struct (snake_case) so a JS-paused run can be resumed in Rust. */
		export interface SpiderCheckpoint {
		pending: Array<{ url: string; method: string; session_id?: string }>;
		seen: string[];
		items_emitted: number;
		}

		export interface RunSpiderOptions {
		/** Override the default node:http(s) fetcher. */
		fetcher?: (req: { url: string; method: string; sessionId?: string; userAgent?: string }) =>
		Promise<SpiderResponse>;
		/** Map<host, robots.txt body>. Required when `robotsTxtObey` is true. */
		robotsCache?: Map<string, string>;
		/** Abort the run (Ctrl-C). Returned handle's `checkpoint()` captures
		* the frontier at the pause point. */
		signal?: AbortSignal;
		/** Resume from a previously captured [`SpiderCheckpoint`]. */
		resume?: SpiderCheckpoint;
		}

		/** Async iterable of items yielded by `parse`. Call `checkpoint()` after
		* the loop terminates to snapshot the frontier for a resumable pause. */
		export interface SpiderHandle extends AsyncIterableIterator<Record<string, unknown>> {
		checkpoint(): SpiderCheckpoint;
		isPaused(): boolean;
		/**
		* Independent broadcast subscriber over the items this spider yields.
		* Calls return a fresh `AsyncIterable<Item>` each invocation — multiple
		* consumers can `stream()` the same run in parallel. Backpressure: a
		* slow subscriber sees the oldest items dropped from its bounded ring
		* buffer (`bufferSize`, default 1024) rather than blocking the
		* producer or crashing the bus. Mirrors the Rust
		* `SpiderRunner::stream()` contract.
		*
		* Items only flow once the primary handle is iterated (or once the
		* driver task has started), so consumers should typically subscribe
		* before the first `await handle.next()` if they want a guaranteed
		* count.
		*/
		stream(opts?: { bufferSize?: number }): AsyncIterableIterator<Record<string, unknown>>;
		}

		/** Validate + freeze a spider spec. */
		export function defineSpider(spec: SpiderSpec): SpiderDef;

		/** Drive a defined spider. */
		export function runSpider(spider: SpiderDef, opts?: RunSpiderOptions): SpiderHandle;

+1

-1

sdk/version.js

		// Auto-generated by scripts/sync-version.js. Do not edit by hand.
		// Single source of truth: package.json.version.
		'use strict';
		module.exports = "1.0.4";
		module.exports = "1.0.6";

crawlex - npm Package Compare versions

New alerts

Improved metrics

Worsened metrics