Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
socks-scraper
Advanced tools
socks-scraper - npm Package Compare versions
Comparing version 1.1.0 to 1.2.0-h
@@ -6,20 +6,20 @@ const SocksScraper = require('.'); // require('socks-scraper'); | ||
| const socksScraper = new SocksScraper([ | ||
| "https://api.proxyscrape.com/?request=displayproxies&status=alive", | ||
| "https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5", | ||
| "https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4", | ||
| "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt", | ||
| "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt", | ||
| "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/http/http.txt", | ||
| "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/socks4/socks4.txt", | ||
| "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/socks5/socks5.txt", | ||
| 'https://api.proxyscrape.com/?request=displayproxies&status=alive&proxytype=socks4', | ||
| 'https://api.proxyscrape.com/?request=displayproxies&status=alive&proxytype=socks5', | ||
| 'https://openproxylist.xyz/socks4.txt', | ||
| 'https://openproxylist.xyz/socks5.txt', | ||
| 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt' | ||
| ]) | ||
| // Add one more site to the list of sites on which free proxies are placed | ||
| socksScraper.addSites(["https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt"]) | ||
| socksScraper.addSites(["https://api.proxyscrape.com/?request=displayproxies&status=alive"]) | ||
| // Timeout for checking the proxy in ms | ||
| const timeout = 6000 | ||
| const timeout = 10000 | ||
| const chunkSize = 5000; | ||
| const retryCount = 5; | ||
| console.log('Updating unchecked proxies...'); | ||
@@ -32,3 +32,3 @@ // Gets proxies from all sites, VERY IMPORTANT: it must be called before the getWorkedSocksProxies() | ||
| // Get a list of proxies from all sites, check if they work and return the best ones | ||
| const wsp4 = await socksScraper.getWorkedSocksProxies('socks4', timeout) | ||
| const wsp4 = await socksScraper.getWorkedSocksProxies('socks4', timeout, chunkSize, undefined, undefined, retryCount) | ||
@@ -38,8 +38,8 @@ // Sort the list by latency and take the fastest proxy | ||
| console.log(`The best socks4 proxy is ${bestWSP4.host}:${bestWSP4.port} with latency ${bestWSP4.latency}ms (${wsp4.length})`) | ||
| console.log(`The best socks4 proxy is ${bestWSP4.address} with latency ${bestWSP4.latency}ms (${wsp4.length})`) | ||
| const wsp5 = await socksScraper.getWorkedSocksProxies('socks5', timeout) | ||
| const wsp5 = await socksScraper.getWorkedSocksProxies('socks5', timeout, chunkSize, undefined, undefined, retryCount) | ||
| const bestWSP5 = SocksScraper.filterByLatency(wsp5)[0] | ||
| console.log(`The best socks5 proxy is ${bestWSP5.host}:${bestWSP5.port} with latency ${bestWSP5.latency}ms (${wsp5.length})`) | ||
| console.log(`The best socks5 proxy is ${bestWSP5.address} with latency ${bestWSP5.latency}ms (${wsp5.length})`) | ||
@@ -46,0 +46,0 @@ /* only if you have VERY good internet... |
43
index.js
@@ -35,3 +35,3 @@ const { ProxyAgent } = require('proxy-agent'); | ||
| function defaultProxyCallback(response) { | ||
| return response.body?.origin || JSON.parse(response.body)?.origin | ||
| return true; | ||
| } | ||
@@ -78,3 +78,3 @@ | ||
| clearSites() { | ||
| this.sites = [] | ||
| this.sites.length = 0; | ||
| } | ||
@@ -93,3 +93,3 @@ | ||
| clearCheckedProxies() { | ||
| this.checkedProxies = []; | ||
| this.sites.length = 0; | ||
| } | ||
@@ -147,6 +147,8 @@ | ||
| * @param {number?} timeout | ||
| * @param {string?} website | ||
| * @param {Function?} callback | ||
| * @param {number?} retryCount | ||
| */ | ||
| static async isAliveProxy(type, address, timeout = 6000, website = 'https://httpbin.org/ip', callback = defaultProxyCallback) { | ||
| static async isAliveProxy(type, address, timeout = 6000, website = 'https://discord.com', callback = defaultProxyCallback, retryCount = 0) { | ||
| try { | ||
| const agent = new ProxyAgent({ | ||
@@ -182,2 +184,6 @@ getProxyForUrl: () => `${type}://${address}`, | ||
| } catch (error) { | ||
| if (retryCount < 1) { | ||
| // Retry once more | ||
| return this.isAliveProxy(type, address, timeout, website, callback, retryCount + 1); | ||
| } | ||
| return false | ||
@@ -202,18 +208,27 @@ } | ||
| * @param {number} timeout | ||
| * @param {number} [chunkSize=1000] | ||
| * @param {string?} website | ||
| * @param {Function?} callback | ||
| * @param {number?} retryCount | ||
| * @returns {Promise<SocksScraper.IDefaultProxy[]>} | ||
| */ | ||
| async getWorkedSocksProxies(sockType, timeout) { | ||
| this.clearCheckedProxies() | ||
| async getWorkedSocksProxies(sockType, timeout, chunkSize = 1000, website = undefined, callback = undefined, retryCount = undefined) { | ||
| const proxyArray = Array.from(this.unCheckedProxies); | ||
| const chunks = []; | ||
| const checkedProxiesPromise = Array.from(this.unCheckedProxies).map(async (a) => SocksScraper.isAliveProxy(sockType, a, timeout)) | ||
| for (let i = 0; i < proxyArray.length; i += chunkSize) { | ||
| chunks.push(proxyArray.slice(i, i + chunkSize)); | ||
| } | ||
| const checkedProxies = await Promise.all(checkedProxiesPromise) | ||
| for (const chunk of chunks) { | ||
| const checkedProxiesPromise = chunk.map((a) => SocksScraper.isAliveProxy(sockType, a, timeout, website, callback, retryCount)); | ||
| const checkedProxies = await Promise.all(checkedProxiesPromise); | ||
| for (const proxy of checkedProxies) { | ||
| if (!proxy) continue | ||
| this.checkedProxies.push(proxy) | ||
| for (const proxy of checkedProxies) { | ||
| if (!proxy) continue; | ||
| this.checkedProxies.push(proxy); | ||
| } | ||
| } | ||
| return this.checkedProxies | ||
| return this.checkedProxies; | ||
| } | ||
@@ -220,0 +235,0 @@ } |
@@ -7,3 +7,3 @@ { | ||
| "name": "socks-scraper", | ||
| "version": "1.1.0", | ||
| "version": "1.2.0h", | ||
| "description": "Uses your references to obtain and verify proxies ", | ||
@@ -10,0 +10,0 @@ "main": "index.js", |
@@ -12,4 +12,4 @@ # Proxy Scraper | ||
| # Lib Dependencies | ||
| - undici | ||
| - socks | ||
| - needle | ||
| - proxy-agent | ||
@@ -30,43 +30,55 @@ # JSDoc | ||
| ```js | ||
| const SocksScraper = require('socks-scraper'); | ||
| // Initialize the scraper with a list of raw sites | ||
| const socksScraper = new SocksScraper([ | ||
| "https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5", | ||
| "https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4", | ||
| "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt", | ||
| "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt", | ||
| "https://shieldcommunity.net/sockets.txt" | ||
| ]) | ||
| const socksScraper = new SocksScraper([ | ||
| "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/http/http.txt", | ||
| "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/socks4/socks4.txt", | ||
| "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/socks5/socks5.txt", | ||
| // Add one more site to the list of sites on which free proxies are placed | ||
| socksScraper.addSites(["https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt"]) | ||
| 'https://api.proxyscrape.com/?request=displayproxies&status=alive&proxytype=socks4', | ||
| 'https://api.proxyscrape.com/?request=displayproxies&status=alive&proxytype=socks5', | ||
| ]) | ||
| // Timeout for checking the proxy in ms | ||
| const timeout = 6000 | ||
| // Add one more site to the list of sites on which free proxies are placed | ||
| socksScraper.addSites(["https://api.proxyscrape.com/?request=displayproxies&status=alive"]) | ||
| console.log('Updating unchecked proxies...'); | ||
| // Gets proxies from all sites, VERY IMPORTANT: it must be called before the getWorkedSocksProxies() | ||
| await socksScraper.updateUncheckedProxies() | ||
| // Timeout for checking the proxy in ms | ||
| const timeout = 10000 | ||
| console.log('Done updating unchecked proxies!'); | ||
| const chunkSize = 5000; | ||
| // Get a list of proxies from all sites, check if they work and return the best ones | ||
| const wsp4 = await socksScraper.getWorkedSocksProxies(4, timeout) | ||
| // Sort the list by latency and take the fastest proxy | ||
| const bestWSP4 = SocksScraper.filterByLatency(wsp4)[0] | ||
| const retryCount = 5; | ||
| console.log(`The best socks4 proxy is ${bestWSP4.host}:${bestWSP4.port} with latency ${bestWSP4.latency}ms`) | ||
| console.log('Updating unchecked proxies...'); | ||
| // Gets proxies from all sites, VERY IMPORTANT: it must be called before the getWorkedSocksProxies() | ||
| await socksScraper.updateUncheckedProxies() | ||
| const wsp5 = await socksScraper.getWorkedSocksProxies(5, timeout, 1, 20000) | ||
| const bestWSP5 = SocksScraper.filterByLatency(wsp5)[0] | ||
| console.log(`Done updating unchecked proxies! (${socksScraper.unCheckedProxies.size})`); | ||
| console.log(`The best socks5 proxy is ${bestWSP5.host}:${bestWSP5.port} with latency ${bestWSP5.latency}ms`) | ||
| // Get a list of proxies from all sites, check if they work and return the best ones | ||
| const wsp4 = await socksScraper.getWorkedSocksProxies('socks4', timeout, chunkSize, undefined, undefined, retryCount) | ||
| // Check my socks5 proxy to see if it works at all | ||
| const mySocks5Proxy = await SocksScraper.checkSocksProxy(5, '94.131.14.66:1080', 4000) | ||
| const isAlive = Boolean(mySocks5Proxy) | ||
| // Sort the list by latency and take the fastest proxy | ||
| const bestWSP4 = SocksScraper.filterByLatency(wsp4)[0] | ||
| console.log(`My socks5 proxy is ${isAlive ? 'alive' : 'dead'}`) | ||
| console.log(mySocks5Proxy) | ||
| console.log(`The best socks4 proxy is ${bestWSP4.address} with latency ${bestWSP4.latency}ms (${wsp4.length})`) | ||
| const wsp5 = await socksScraper.getWorkedSocksProxies('socks5', timeout, chunkSize, undefined, undefined, retryCount) | ||
| const bestWSP5 = SocksScraper.filterByLatency(wsp5)[0] | ||
| console.log(`The best socks5 proxy is ${bestWSP5.address} with latency ${bestWSP5.latency}ms (${wsp5.length})`) | ||
| /* only if you have VERY good internet... | ||
| const http = await socksScraper.getWorkedSocksProxies('http', timeout) | ||
| const bestHttp = SocksScraper.filterByLatency(http)[0] | ||
| console.log(`The best http proxy is ${bestHttp.host}:${bestHttp.port} with latency ${bestHttp.latency}ms`) | ||
| */ | ||
| // Check my socks5 proxy to see if it works at all | ||
| const mySocks4Proxy = await SocksScraper.isAliveProxy('socks4', '3.10.93.50:80', 10000) | ||
| const isAlive = Boolean(mySocks4Proxy) | ||
| console.log(`My socks4 proxy is ${isAlive ? 'alive' : 'dead'}`) | ||
| console.log(mySocks4Proxy); | ||
| ``` | ||
@@ -73,0 +85,0 @@ ```js |
Sorry, the diff of this file is not supported yet
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Manifest confusion
Supply chain riskThis package has inconsistent metadata. This could be malicious or caused by an error when publishing the package.
Found 1 instance in 1 package
No v1
QualityPackage is not semver >=1. This means it is not stable and does not support ^ ranges.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by4.48%
24420
- Lines of code
- increased by6.01%
247
- Number of lines in readme file
- increased by15.19%
91
Worsened metrics
- Number of low quality alerts
- increased byInfinity%
1
No dependency changes