@sharpapi/sharpapi-node-web-scraping
Advanced tools
SharpAPI Web Scraping fetches and extracts content from web pages, providing structured data including page metadata, content, links, and more in a machine-readable JSON format. Perfect for data collection, content aggregation, SEO analysis, and research.
```bash npm install @sharpapi/sharpapi-node-web-scraping ```
Visit SharpAPI.com to get your API key.
```javascript const { SharpApiWebScrapingService } = require('@sharpapi/sharpapi-node-web-scraping');
const apiKey = process.env.SHARP_API_KEY; // Store your API key in environment variables const service = new SharpApiWebScrapingService(apiKey);
async function scrapeWebsite() { try { // Scrape a webpage const data = await service.scrapeUrl('https://sharpapi.com');
console.log('Title:', data.title);
console.log('Description:', data.meta_description);
console.log('Keywords:', data.meta_keywords);
console.log('Content length:', data.content.length);
console.log('Links found:', data.links.length);
} catch (error) { console.error('Error:', error.message); } }
scrapeWebsite(); ```
This endpoint is synchronous and returns data immediately (no polling required).
scrapeUrl(url)Scrape a webpage and extract its content in structured format.
Parameters:
url (string, required): The URL to scrape (e.g., 'https://example.com' or 'example.com')Returns: Promise - Structured page data with metadata, content, and links
Extracts:
Example: ```javascript const data = await service.scrapeUrl('https://example.com'); console.log('Page Title:', data.title); console.log('Main Content:', data.content); console.log('All Links:', data.links); ```
```javascript const { SharpApiWebScrapingService } = require('@sharpapi/sharpapi-node-web-scraping');
const service = new SharpApiWebScrapingService(process.env.SHARP_API_KEY);
async function scrapeExample() { const result = await service.scrapeUrl('https://sharpapi.com');
console.log('=== Page Information ==='); console.log('Title:', result.title); console.log('Description:', result.meta_description); console.log('Language:', result.language); console.log('Canonical URL:', result.canonical_url);
console.log('\n=== Content ==='); console.log('Characters:', result.content.length); console.log('Preview:', result.content.substring(0, 200) + '...');
console.log('\n=== Links Found ==='); console.log('Total links:', result.links.length); result.links.slice(0, 5).forEach(link => { console.log(`- ${link.text || 'No text'}: ${link.url}`); }); }
scrapeExample(); ```
```javascript const service = new SharpApiWebScrapingService(process.env.SHARP_API_KEY);
async function getSocialMetadata(url) { const data = await service.scrapeUrl(url);
console.log('=== Open Graph Tags ==='); console.log('OG:Title:', data.og_title); console.log('OG:Description:', data.og_description); console.log('OG:Image:', data.og_image); console.log('OG:Type:', data.og_type);
console.log('\n=== Twitter Card ==='); console.log('Card Type:', data.twitter_card); console.log('Title:', data.twitter_title); console.log('Description:', data.twitter_description); console.log('Image:', data.twitter_image); }
getSocialMetadata('https://example.com/article'); ```
```javascript const service = new SharpApiWebScrapingService(process.env.SHARP_API_KEY);
async function analyzeSEO(url) { const data = await service.scrapeUrl(url);
console.log('=== SEO Analysis ==='); console.log('Title:', data.title, `(${data.title.length} chars)`); console.log('Meta Description:', data.meta_description); console.log('Keywords:', data.meta_keywords); console.log('Canonical URL:', data.canonical_url); console.log('Language:', data.language);
console.log('\n=== Headings Structure ==='); if (data.headings) { data.headings.forEach(heading => { console.log(`${heading.level}: ${heading.text}`); }); }
console.log('\n=== Link Analysis ==='); const internalLinks = data.links.filter(l => l.type === 'internal'); const externalLinks = data.links.filter(l => l.type === 'external'); console.log(`Internal links: ${internalLinks.length}`); console.log(`External links: ${externalLinks.length}`); }
analyzeSEO('https://your-website.com'); ```
```javascript const service = new SharpApiWebScrapingService(process.env.SHARP_API_KEY);
async function extractForAI(url) { const data = await service.scrapeUrl(url);
// Extract clean content for AI processing const cleanContent = { title: data.title, description: data.meta_description, mainContent: data.content, language: data.language, author: data.author, publishedDate: data.published_date, modifiedDate: data.modified_date };
console.log('Extracted content ready for AI processing:'); console.log(JSON.stringify(cleanContent, null, 2));
// Now you can pass this to SharpAPI AI endpoints: // - Summarization: @sharpapi/sharpapi-node-summarize-text // - Translation: @sharpapi/sharpapi-node-translate // - Keywords: @sharpapi/sharpapi-node-generate-keywords // - SEO Tags: @sharpapi/sharpapi-node-seo-tags }
extractForAI('https://blog.example.com/article'); ```
```javascript const service = new SharpApiWebScrapingService(process.env.SHARP_API_KEY);
async function analyzeCompetitor(url) { const data = await service.scrapeUrl(url);
console.log('=== Competitor Analysis ==='); console.log('Domain:', new URL(url).hostname); console.log('Title Strategy:', data.title); console.log('Description:', data.meta_description); console.log('Keywords Focus:', data.meta_keywords);
console.log('\n=== Content Strategy ==='); console.log('Content Length:', data.content.length, 'characters'); console.log('Word Count (approx):', Math.round(data.content.split(' ').length));
console.log('\n=== Link Building ==='); const externalLinks = data.links.filter(l => l.type === 'external'); console.log('External Links:', externalLinks.length); externalLinks.slice(0, 10).forEach(link => { console.log(` - ${link.url}`); }); }
analyzeCompetitor('https://competitor-website.com'); ```
The API returns a comprehensive JSON object with the following structure:
```json { "url": "https://sharpapi.com/", "title": "SharpAPI - AI-Powered Workflow Automation API", "meta_description": "Automate workflows with AI-powered API...", "meta_keywords": "AI API, automation, workflow", "author": "SharpAPI Team", "language": "en", "charset": "UTF-8", "canonical_url": "https://sharpapi.com/", "viewport": "width=device-width, initial-scale=1",
"og_title": "SharpAPI - AI-Powered API", "og_description": "Automate your workflows...", "og_image": "https://sharpapi.com/og-image.jpg", "og_type": "website", "og_url": "https://sharpapi.com/",
"twitter_card": "summary_large_image", "twitter_title": "SharpAPI", "twitter_description": "AI-Powered API", "twitter_image": "https://sharpapi.com/twitter-card.jpg",
"content": "Full page content as text...", "text_content": "Clean text without HTML...",
"headings": [ { "level": "h1", "text": "Main Heading" }, { "level": "h2", "text": "Subheading" } ],
"links": [ { "url": "https://sharpapi.com/about", "text": "About Us", "type": "internal", "rel": null }, { "url": "https://example.com", "text": "External Link", "type": "external", "rel": "nofollow" } ],
"images": [ { "src": "https://sharpapi.com/image.jpg", "alt": "Image description" } ],
"extracted_at": "2026-01-10T15:30:00Z", "processing_time_ms": 1250 } ```
The extracted data can be seamlessly integrated with SharpAPI's AI-powered endpoints for further analysis:
```javascript const { SharpApiWebScrapingService } = require('@sharpapi/sharpapi-node-web-scraping'); const { SharpApiSummarizeService } = require('@sharpapi/sharpapi-node-summarize-text');
const scrapingService = new SharpApiWebScrapingService(process.env.SHARP_API_KEY); const summarizeService = new SharpApiSummarizeService(process.env.SHARP_API_KEY);
async function scrapeAndSummarize(url) { // 1. Scrape the webpage const scraped = await scrapingService.scrapeUrl(url);
// 2. Summarize the content const statusUrl = await summarizeService.summarize(scraped.content); const summary = await summarizeService.fetchResults(statusUrl);
console.log('Original length:', scraped.content.length); console.log('Summary:', summary.getResultJson()); }
scrapeAndSummarize('https://blog.example.com/long-article'); ```
GET /utilities/scrape_url?url={url}
This endpoint is synchronous and returns 200 OK immediately.
For detailed API specifications, refer to:
This project is licensed under the MIT License. See the LICENSE.md file for details.
Powered by SharpAPI - AI-Powered API Workflow Automation
FAQs
SharpAPI.com Node.js SDK for Web Scraping API
The npm package @sharpapi/sharpapi-node-web-scraping receives a total of 2 weekly downloads. As such, @sharpapi/sharpapi-node-web-scraping popularity was classified as not popular.
We found that @sharpapi/sharpapi-node-web-scraping demonstrated a healthy version release cadence and project activity because the last version was released less than a year ago. It has 1 open source maintainer collaborating on the project.
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Security News
Socket CEO Feross Aboukhadijeh joins 10 Minutes or Less, a podcast by Ali Rohde, to discuss the recent surge in open source supply chain attacks.
Research
/Security News
Campaign of 108 extensions harvests identities, steals sessions, and adds backdoors to browsers, all tied to the same C2 infrastructure.
Security News
OpenAI rotated macOS signing certificates after a malicious Axios package reached its CI pipeline in a broader software supply chain attack.
