🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

@scrapio/mcp

Package Overview
Dependencies
Maintainers
1
Versions
5
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@scrapio/mcp - npm Package Compare versions

Comparing version
1.0.1
 to
1.0.2
+5
-1
package.json
{
"name": "@scrapio/mcp",
"version": "1.0.1",
"version": "1.0.2",
"description": "Official MCP server for the Scrapio — fetch, search, crawl, and interact from any AI agent",

@@ -26,3 +26,7 @@ "type": "module",

},
"files": [
"dist",
"README.md"
],
"license": "MIT"
}
-15
src/auth.ts
import { ApiClient } from "@scrapio/api";
export function createClient(): ApiClient {
const apiKey = process.env.SCRAPIO_API_KEY;
if (!apiKey) {
process.stderr.write(
"Error: SCRAPIO_API_KEY is not set.\n" +
"Get your API key at https://app.scrapio.dev/settings/api-keys\n"
);
process.exit(1);
}
const baseUrl = process.env.SCRAPIO_BASE_URL;
return new ApiClient({ apiKey, ...(baseUrl ? { baseUrl } : {}) });
}
-36
src/errors.ts
import { ApiError, AuthError, RateLimitError, CreditsExhaustedError } from "@scrapio/api";
export function toToolError(err: unknown): { isError: true; content: [{ type: "text"; text: string }] } {
let text: string;
if (err instanceof AuthError) {
text = "Authentication failed. Check your SCRAPIO_API_KEY.";
} else if (err instanceof CreditsExhaustedError) {
text = "Credits exhausted. Add credits at app.scrapio.dev.";
} else if (err instanceof RateLimitError) {
text = "Rate limit reached. Wait a moment and try again.";
} else if (err instanceof ApiError) {
switch (err.statusCode) {
case 502:
text = "The target site blocked the request. Try again or use the interact tool for JavaScript-heavy pages.";
break;
case 504:
text = "Request timed out. Increase timeout_ms or use submit_job for async execution.";
break;
default:
text = `API error (${err.statusCode}): ${err.message}`;
}
} else if (err instanceof Error) {
if (err.message.includes("timed out")) {
text = "Request timed out. Increase timeout_ms or use submit_job for async execution.";
} else if (err.message.includes("fetch") || err.message.includes("network") || err.message.includes("ECONNREFUSED")) {
text = "Failed to reach the API. Check your network connection.";
} else {
text = err.message;
}
} else {
text = "An unexpected error occurred.";
}
return { isError: true, content: [{ type: "text", text }] };
}
-47
src/index.ts
#!/usr/bin/env node
import { createClient } from "./auth.js";
import { createServer } from "./server.js";
import { startStdio } from "./transport/stdio.js";
import { startHttp } from "./transport/http.js";
function parseArgs(): { transport: "stdio" | "http"; port: number } {
const args = process.argv.slice(2);
let transport: "stdio" | "http" = "stdio";
let port = Number(process.env.MCP_PORT ?? 3010);
for (let i = 0; i < args.length; i++) {
if (args[i] === "--transport" && args[i + 1]) {
const t = args[++i];
if (t !== "stdio" && t !== "http") {
process.stderr.write(`Error: --transport must be "stdio" or "http"\n`);
process.exit(1);
}
transport = t;
} else if (args[i] === "--port" && args[i + 1]) {
port = Number(args[++i]);
if (isNaN(port)) {
process.stderr.write("Error: --port must be a number\n");
process.exit(1);
}
}
}
return { transport, port };
}
async function main(): Promise<void> {
const { transport, port } = parseArgs();
const client = createClient();
const server = createServer(client);
if (transport === "http") {
await startHttp(server, port);
} else {
await startStdio(server);
}
}
main().catch((err) => {
process.stderr.write(`Fatal: ${err instanceof Error ? err.message : String(err)}\n`);
process.exit(1);
});
-30
src/server.ts
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { registerFetchTool } from "./tools/fetch.js";
import { registerGoogleSearchTool } from "./tools/google-search.js";
import { registerYoutubeTranscriptTool } from "./tools/youtube-transcript.js";
import { registerAmazonProductTool } from "./tools/amazon-product.js";
import { registerWalmartSearchTool } from "./tools/walmart-search.js";
import { registerInteractTool } from "./tools/interact.js";
import { registerCrawlTool } from "./tools/crawl.js";
import { registerSubmitJobTool } from "./tools/submit-job.js";
import { registerGetJobTool } from "./tools/get-job.js";
export function createServer(client: ApiClient): McpServer {
const server = new McpServer({
name: "scrapio",
version: "1.0.0",
});
registerFetchTool(server, client);
registerGoogleSearchTool(server, client);
registerYoutubeTranscriptTool(server, client);
registerAmazonProductTool(server, client);
registerWalmartSearchTool(server, client);
registerInteractTool(server, client);
registerCrawlTool(server, client);
registerSubmitJobTool(server, client);
registerGetJobTool(server, client);
return server;
}
-45
src/tools/amazon-product.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
export function registerAmazonProductTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"amazon_product",
{
description:
"Get structured product data from Amazon. " +
"Provide a product URL, an ASIN, or a search query — one is required. " +
"Returns title, price, rating, availability, images, and bullet points.",
inputSchema: {
url: z.string().optional().describe("Full Amazon product URL."),
asin: z.string().optional().describe('Amazon Standard Identification Number (e.g. "B08N5WRWNW").'),
query: z.string().optional().describe("Search query to find a product. Returns the first matching result."),
country: z.string().optional().describe(
'Two-letter country code for the Amazon marketplace (e.g. "us", "uk", "de"). Defaults to "us".'
),
},
},
async (args) => {
try {
if (!args.url && !args.asin && !args.query) {
return {
isError: true,
content: [{ type: "text", text: "Provide at least one of: url, asin, or query." }],
};
}
const params = args.asin
? { asin: args.asin, country: args.country }
: args.url
? { asin: args.url, country: args.country }
: { asin: args.query!, country: args.country };
const result = await client.amazon.getProduct(params);
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-77
src/tools/crawl.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
export function registerCrawlTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"crawl",
{
description:
"Crawl one or more seed URLs and return the content of every page visited. " +
"Follows links within the same domain up to a configurable depth and page count. " +
"Use fetch for a single known URL. Use crawl when you need content from multiple pages " +
"and don't want to enumerate each URL manually.",
inputSchema: {
seeds: z
.array(z.string())
.min(1)
.describe("One or more starting URLs. The crawler follows links discovered from these pages."),
max_pages: z
.number()
.int()
.min(1)
.max(50)
.optional()
.describe("Maximum number of pages to visit. Defaults to 10."),
max_depth: z
.number()
.int()
.min(0)
.max(5)
.optional()
.describe(
"Maximum link depth from a seed URL. 0 means seed pages only; 1 means seeds plus one hop. Defaults to 2."
),
same_domain_only: z
.boolean()
.optional()
.describe("Whether to restrict crawling to the same domain as seeds. Defaults to true."),
output: z
.array(z.enum(["markdown", "html", "json"]))
.optional()
.describe('Output format for each page. Defaults to ["markdown"].'),
extract: z
.object({
mode: z.enum(["schema", "instruction"]),
schema: z.record(z.string(), z.unknown()).optional(),
instruction: z.string().optional(),
})
.optional()
.describe("Optional structured extraction applied to every page. Requires output to include 'json'."),
timeout_ms: z
.number()
.int()
.positive()
.optional()
.describe("Maximum total time for the entire crawl in milliseconds."),
},
},
async (args) => {
try {
const result = await client.crawl.crawl({
seeds: args.seeds,
max_pages: args.max_pages,
max_depth: args.max_depth,
same_domain_only: args.same_domain_only,
output: args.output as string[] | undefined,
extract: args.extract as Record<string, unknown> | undefined,
timeout_ms: args.timeout_ms,
});
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-63
src/tools/fetch.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
export function registerFetchTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"fetch",
{
description:
"Fetch a URL and return its content. Renders JavaScript by default. " +
"Returns markdown by default; also supports html, json extraction, and screenshots.",
inputSchema: {
url: z.string().describe("The URL to fetch. Must be http or https."),
render_js: z.boolean().optional().describe("Whether to render JavaScript before extracting content. Defaults to true."),
output: z
.array(z.enum(["markdown", "html", "json", "screenshot"]))
.optional()
.describe('Output formats to return. Defaults to ["markdown"].'),
extract: z
.object({
mode: z.enum(["schema", "instruction"]).describe(
"schema: extract fields from a JSON schema. instruction: follow a natural-language instruction."
),
schema: z.record(z.string(), z.unknown()).optional().describe(
"Key-value map of field names to types. Required when mode is schema."
),
instruction: z.string().optional().describe(
"Natural-language instruction for what to extract. Required when mode is instruction."
),
})
.optional()
.describe("Optional structured extraction config. Requires output to include 'json'."),
device: z
.enum(["desktop", "mobile", "tablet"])
.optional()
.describe("Device type to emulate. Defaults to desktop."),
timeout_ms: z.number().int().positive().optional().describe(
"Maximum time to wait in milliseconds."
),
},
},
async (args) => {
try {
const result = await client.fetch.fetch({
url: args.url,
render_js: args.render_js,
output: args.output as string[] | undefined,
extract: args.extract as Record<string, unknown> | undefined,
device: args.device,
});
const outputs = result.outputs;
if (typeof outputs.markdown === "string") {
return { content: [{ type: "text", text: outputs.markdown }] };
}
return { content: [{ type: "text", text: JSON.stringify(outputs, null, 2) }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-51
src/tools/get-job.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
const TERMINAL_STATUSES = new Set(["completed", "partial", "failed", "cancelled"]);
export function registerGetJobTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"get_job",
{
description:
"Check the status of an async job submitted via submit_job. " +
"When the job is complete, returns the full result inline. " +
"Poll this tool every few seconds until status is 'completed', 'failed', or 'cancelled'.",
inputSchema: {
job_id: z.string().describe("The job ID returned by submit_job."),
},
},
async (args) => {
try {
const job = await client.jobs.get(args.job_id);
if (TERMINAL_STATUSES.has(job.status)) {
const result = await client.jobs.getResult(args.job_id);
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
}
return {
content: [
{
type: "text",
text: JSON.stringify(
{
job_id: job.job_id,
status: job.status,
created_at: job.created_at,
updated_at: job.updated_at,
},
null,
2
),
},
],
};
} catch (err) {
return toToolError(err);
}
}
);
}
-49
src/tools/google-search.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
export function registerGoogleSearchTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"google_search",
{
description:
"Search Google and return structured results. " +
"Supports web, news, images, shopping, and maps search types.",
inputSchema: {
query: z.string().describe("The search query."),
search_type: z
.enum(["classic", "news", "images", "shopping", "maps"])
.optional()
.describe("Type of search to run. Defaults to classic."),
country_code: z.string().optional().describe(
'Two-letter ISO country code for localized results (e.g. "us", "gb").'
),
language: z.string().optional().describe('Language for results (e.g. "en", "fr").'),
date_range: z
.enum(["past_hour", "past_day", "past_week", "past_month", "past_year"])
.optional()
.describe("Filter results to a time range."),
device: z
.enum(["desktop", "mobile"])
.optional()
.describe("Device type for results. Defaults to desktop."),
},
},
async (args) => {
try {
const result = await client.google.search({
search: args.query,
search_type: args.search_type as never,
country_code: args.country_code,
language: args.language,
date_range: args.date_range as never,
device: args.device,
});
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-94
src/tools/interact.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import type { InteractAction } from "@scrapio/api";
import { toToolError } from "../errors.js";
const actionSchema = z.discriminatedUnion("type", [
z.object({
type: z.literal("goto"),
url: z.string().describe("URL to navigate to."),
}),
z.object({
type: z.literal("click"),
selector: z.string().describe("CSS selector of the element to click."),
}),
z.object({
type: z.literal("type"),
selector: z.string().describe("CSS selector of the input element."),
value: z.string().describe("Text to type into the element."),
}),
z.object({
type: z.literal("select"),
selector: z.string().describe("CSS selector of the select element."),
value: z.string().describe("Option value to select."),
}),
z.object({
type: z.literal("press"),
key: z.string().describe('Key to press (e.g. "Enter", "Tab").'),
selector: z.string().optional().describe("CSS selector to focus before pressing."),
}),
z.object({
type: z.literal("scroll"),
direction: z.enum(["down", "up"]).optional().describe("Scroll direction. Defaults to down."),
amount: z.number().int().optional().describe("Pixels to scroll."),
selector: z.string().optional().describe("Element to scroll within."),
}),
z.object({
type: z.literal("wait_for"),
selector: z.string().describe("CSS selector to wait for before proceeding."),
}),
z.object({
type: z.literal("wait_ms"),
duration_ms: z.number().int().positive().describe("Milliseconds to wait."),
}),
]);
export function registerInteractTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"interact",
{
description:
"Drive a real browser through a multi-step workflow. " +
"Use this when a page requires login, clicking through steps, filling forms, or navigating dynamic UI. " +
"Returns the final page content after all actions complete.",
inputSchema: {
url: z.string().describe("Starting URL for the browser session."),
actions: z
.array(actionSchema)
.min(1)
.describe("Ordered list of actions to perform."),
output: z
.array(z.enum(["markdown", "html", "screenshot"]))
.optional()
.describe('Output formats to capture after the final action. Defaults to ["markdown"].'),
device: z
.enum(["desktop", "mobile", "tablet"])
.optional()
.describe("Device type to emulate. Defaults to desktop."),
timeout_ms: z.number().int().positive().optional().describe(
"Maximum total time for the entire workflow in milliseconds."
),
},
},
async (args) => {
try {
const result = await client.interact.interact({
url: args.url,
actions: args.actions as InteractAction[],
output: args.output as string[] | undefined,
device: args.device,
timeout_ms: args.timeout_ms,
});
const outputs = result.outputs;
if (typeof outputs.markdown === "string") {
return { content: [{ type: "text", text: outputs.markdown }] };
}
return { content: [{ type: "text", text: JSON.stringify(outputs, null, 2) }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-54
src/tools/submit-job.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
export function registerSubmitJobTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"submit_job",
{
description:
"Submit a long-running job asynchronously and return a job ID. " +
"Use this instead of fetch or crawl when the work takes more than a few seconds. " +
"After submitting, call get_job with the returned job_id to check status and retrieve the result.",
inputSchema: {
kind: z
.enum(["fetch", "interact", "search", "crawl", "map"])
.describe("Type of job to submit."),
input: z
.record(z.string(), z.unknown())
.describe(
"The job input payload. Must match the schema for the chosen kind " +
"(same as the equivalent synchronous tool input)."
),
webhook_url: z
.string()
.optional()
.describe("Optional URL to receive a POST callback when the job completes."),
},
},
async (args) => {
try {
const job = await client.jobs.create({
job_type: args.kind,
payload: args.input,
webhook_url: args.webhook_url,
});
return {
content: [
{
type: "text",
text: JSON.stringify(
{ job_id: job.job_id, status: job.status, created_at: job.created_at },
null,
2
),
},
],
};
} catch (err) {
return toToolError(err);
}
}
);
}
-26
src/tools/walmart-search.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
export function registerWalmartSearchTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"walmart_search",
{
description:
"Search Walmart and return structured product listings including prices, ratings, and availability.",
inputSchema: {
query: z.string().describe('Search query (e.g. "noise cancelling headphones").'),
delivery_zip: z.string().optional().describe("ZIP code for availability and delivery information."),
},
},
async (args) => {
try {
const result = await client.walmart.search({ query: args.query });
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-65
src/tools/youtube-transcript.ts
import { z } from "zod";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { ApiClient } from "@scrapio/api";
import { toToolError } from "../errors.js";
function extractVideoId(url: string): string {
try {
const parsed = new URL(url);
if (parsed.hostname === "youtu.be") {
return parsed.pathname.slice(1);
}
const v = parsed.searchParams.get("v");
if (v) return v;
} catch {
// not a URL — treat as a raw video ID
}
return url;
}
function formatTranscript(subtitles: unknown[]): string {
if (!subtitles.length) return "(No transcript available for this video.)";
return subtitles
.map((s) => {
const sub = s as Record<string, unknown>;
const start = typeof sub.start === "number" ? sub.start : 0;
const h = Math.floor(start / 3600);
const m = Math.floor((start % 3600) / 60);
const sec = Math.floor(start % 60);
const ts = [h, m, sec].map((n) => String(n).padStart(2, "0")).join(":");
return `[${ts}] ${sub.text ?? ""}`;
})
.join("\n");
}
export function registerYoutubeTranscriptTool(server: McpServer, client: ApiClient): void {
server.registerTool(
"youtube_transcript",
{
description:
"Get the transcript (subtitles/captions) for a YouTube video. " +
"Accepts a full YouTube URL or a bare video ID. Returns the transcript as plain text with timestamps.",
inputSchema: {
url: z.string().describe(
"Full YouTube video URL (e.g. https://www.youtube.com/watch?v=...) or bare video ID."
),
language: z.string().optional().describe(
'Preferred transcript language code (e.g. "en", "es"). Returns the best available match. Defaults to en.'
),
},
},
async (args) => {
try {
const video_id = extractVideoId(args.url);
const result = await client.youtube.getSubtitles({
video_id,
language: args.language ?? "en",
});
const text = formatTranscript(result.subtitles as unknown[]);
return { content: [{ type: "text", text }] };
} catch (err) {
return toToolError(err);
}
}
);
}
-38
src/transport/http.ts
import http from "node:http";
import { randomUUID } from "node:crypto";
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
export async function startHttp(server: McpServer, port: number): Promise<void> {
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => randomUUID(),
});
await server.connect(transport);
const httpServer = http.createServer(async (req, res) => {
if (req.url === "/mcp" || req.url === "/mcp/") {
let body: unknown;
if (req.method === "POST") {
body = await new Promise((resolve, reject) => {
let data = "";
req.on("data", (chunk: Buffer) => { data += chunk.toString(); });
req.on("end", () => {
try { resolve(JSON.parse(data)); } catch { resolve(undefined); }
});
req.on("error", reject);
});
}
await transport.handleRequest(req, res, body);
} else if (req.url === "/health") {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ status: "ok" }));
} else {
res.writeHead(404);
res.end();
}
});
await new Promise<void>((resolve) => httpServer.listen(port, resolve));
process.stderr.write(`Scrapio MCP server running on HTTP port ${port} (endpoint: /mcp)\n`);
}
-8
src/transport/stdio.ts
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
export async function startStdio(server: McpServer): Promise<void> {
const transport = new StdioServerTransport();
await server.connect(transport);
process.stderr.write("Scrapio MCP server running on stdio\n");
}
-15
tsconfig.json
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"lib": ["ES2022"],
"types": ["node"],
"strict": true,
"outDir": "dist",
"rootDir": "src",
"esModuleInterop": true,
"skipLibCheck": true
},
"include": ["src"]
}