New Case Study:See how Anthropic automated 95% of dependency reviews with Socket.Learn More →
socks-scraper
Advanced tools
socks-scraper - npm Package Compare versions
Comparing version 1.0.3 to 1.0.4
96
index.js
| const { SocksClient } = require('socks'); | ||
| const { request } = require('undici'); | ||
| const { Worker } = require('node:worker_threads'); | ||
| const path = require('path') | ||
| const workerPath = path.join(__dirname, 'worker.js') | ||
| const ipPortRegex = /^(?:\d{1,3}\.){3}\d{1,3}:\d+$/; | ||
@@ -17,5 +20,11 @@ | ||
| * @typedef {Object} SocksScraper.ClassOptions | ||
| * @property {?string} checkURL | ||
| * @property {?number} checkURLPort | ||
| * @property {?SocksScraper.IncomingHttpHeaders} headers | ||
| */ | ||
| /** | ||
| * @typedef {Object} IDefaultMessage | ||
| * @property {SocksScraper.SocksProxyType} sockType | ||
| * @property {number} timeout | ||
| * @property {string[]} proxyChunk | ||
| * @property {number} chunkSize | ||
| */ | ||
@@ -37,3 +46,3 @@ | ||
| */ | ||
| constructor(sites = [], options = { checkURL: null, checkURLPort: null, headers: null }) { | ||
| constructor(sites = [], options = { headers: null }) { | ||
@@ -48,18 +57,10 @@ /** | ||
| * @private | ||
| * @type {typeof options.checkURL} | ||
| */ | ||
| this.checkURL = options?.checkURL || 'http://ifconfig.me/ip' | ||
| this.unCheckedProxies = [] | ||
| /** | ||
| * @private | ||
| * @type {typeof options.checkURLPort} | ||
| */ | ||
| this.checkURLPort = options?.checkURLPort || 80 | ||
| /** | ||
| * @private | ||
| * @type {typeof options.headers} | ||
| */ | ||
| this.headers = options?.headers || { | ||
| "content-type": "application/x-www-form-urlencoded", | ||
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0', | ||
@@ -109,6 +110,5 @@ 'Connection': 'keep-alive' | ||
| const proxyList = responseText.split('\n').filter((x) => x.match(ipPortRegex)) | ||
| return proxyList | ||
| } catch (error) { | ||
| return [] | ||
| return null | ||
| } | ||
@@ -125,18 +125,18 @@ } | ||
| */ | ||
| async checkSocksProxy(type, address, timeout = 5000) { | ||
| static async checkSocksProxy(type, address, timeout = 5000, checkURL = 'http://ip-api.com/ip', checkURLPort = 80) { | ||
| const [host, portStr] = address.split(':'); | ||
| const port = Number(portStr); | ||
| const startTime = performance.now(); | ||
| try { | ||
| const [host, portStr] = address.split(':'); | ||
| const port = Number(portStr); | ||
| const startTime = performance.now(); | ||
| const result = await SocksClient.createConnection({ | ||
| await SocksClient.createConnection({ | ||
| timeout, | ||
| command: 'connect', | ||
| destination: { host: this.checkURL, port: this.checkURLPort }, | ||
| destination: { host: checkURL, port: checkURLPort }, | ||
| proxy: { host, port, type } | ||
| }) | ||
| .then(() => ({ address, host, port, latency: Math.round(performance.now() - startTime) })) | ||
| .catch(() => null); | ||
| const latency = Math.round(performance.now() - startTime) | ||
| return result; | ||
| return { address, host, port, latency } | ||
| } catch (error) { | ||
@@ -148,2 +148,13 @@ return null | ||
| /** | ||
| * @public | ||
| * @returns {Promise<void>} | ||
| */ | ||
| async updateUncheckedProxies() { | ||
| const sitesPromise = this.sites.map(async (siteUrl) => await this.getProxiesFromRawSite(siteUrl)) | ||
| const notTestedProxyList = await Promise.all(sitesPromise) | ||
| this.unCheckedProxies = Array.prototype.concat(...notTestedProxyList) | ||
| } | ||
| /** | ||
| * Get proxies from all added sites, check if they work and return them as SocksScraper.IDefaultProxy[] | ||
@@ -153,20 +164,31 @@ * @public | ||
| * @param {number} timeout | ||
| * @returns {Promise<SocksScraper.IDefaultProxy[]>} | ||
| * @param {number} [chunkSize=10000] number of Promise.all treatments for 1 worker | ||
| * @param {number} [workerCount=1] count of workers | ||
| * @returns {Promise<?SocksScraper.IDefaultProxy[]>} | ||
| */ | ||
| async getWorkedSocksProxies(sockType, timeout) { | ||
| const sitesPromise = this.sites.map(async (siteUrl) => { | ||
| return await this.getProxiesFromRawSite(siteUrl) | ||
| }) | ||
| async getWorkedSocksProxies(sockType, timeout, workerCount = 1, chunkSize = 10000) { | ||
| const workers = [], workedProxyLists = []; | ||
| const proxyChunkSize = Math.ceil(this.unCheckedProxies.length / workerCount); | ||
| const notTestedProxyList = await Promise.all(sitesPromise) | ||
| for (let i = 0; i < workerCount; i++) { | ||
| const worker = new Worker(workerPath); | ||
| const NTPListPromise = notTestedProxyList.flat().flat() | ||
| .map(async (proxyAddress) => { | ||
| return await this.checkSocksProxy(sockType, proxyAddress, timeout) | ||
| }) | ||
| worker.on('message', (workedProxyList) => { | ||
| workedProxyLists.push(...workedProxyList); | ||
| }); | ||
| const workedProxyList = await Promise.all(NTPListPromise) | ||
| const clearList = workedProxyList.filter((proxy) => Boolean(proxy)) | ||
| const proxyChunk = this.unCheckedProxies.slice(i * proxyChunkSize, (i + 1) * proxyChunkSize); | ||
| worker.postMessage({ sockType, timeout, proxyChunk, chunkSize }); | ||
| workers.push(worker); | ||
| } | ||
| return clearList | ||
| await Promise.all(workers.map((worker) => new Promise((resolve) => { | ||
| worker.once('message', resolve); | ||
| }))); | ||
| for (const worker of workers) { | ||
| worker.terminate(); | ||
| } | ||
| return workedProxyLists; | ||
| } | ||
@@ -173,0 +195,0 @@ } |
@@ -7,3 +7,3 @@ { | ||
| "name": "socks-scraper", | ||
| "version": "1.0.3", | ||
| "version": "1.0.4", | ||
| "description": "Uses your references to obtain and verify proxies ", | ||
@@ -10,0 +10,0 @@ "main": "index.js", |
@@ -35,5 +35,5 @@ # Proxy Scraper | ||
| "https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4", | ||
| "https://shieldcommunity.net/sockets.txt", | ||
| "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt", | ||
| "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt" | ||
| "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt", | ||
| "https://shieldcommunity.net/sockets.txt" | ||
| ]) | ||
@@ -47,2 +47,8 @@ | ||
| console.log('Updating unchecked proxies...'); | ||
| // Gets proxies from all sites, VERY IMPORTANT: it must be called before the getWorkedSocksProxies() | ||
| await socksScraper.updateUncheckedProxies() | ||
| console.log('Done updating unchecked proxies!'); | ||
| // Get a list of proxies from all sites, check if they work and return the best ones | ||
@@ -55,3 +61,3 @@ const wsp4 = await socksScraper.getWorkedSocksProxies(4, timeout) | ||
| const wsp5 = await socksScraper.getWorkedSocksProxies(5, timeout) | ||
| const wsp5 = await socksScraper.getWorkedSocksProxies(5, timeout, 1, 20000) | ||
| const bestWSP5 = SocksScraper.filterByLatency(wsp5)[0] | ||
@@ -62,3 +68,3 @@ | ||
| // Check my socks5 proxy to see if it works at all | ||
| const mySocks5Proxy = await socksScraper.checkSocksProxy(5, '3.122.84.99:3128', 4000) | ||
| const mySocks5Proxy = await SocksScraper.checkSocksProxy(5, '94.131.14.66:1080', 4000) | ||
| const isAlive = Boolean(mySocks5Proxy) | ||
@@ -72,7 +78,7 @@ | ||
| { | ||
| address: '3.122.84.99:3128', | ||
| host: '3.122.84.99', | ||
| port: 3128, | ||
| latency: 32 | ||
| address: '3.122.84.99:3128', | ||
| host: '3.122.84.99', | ||
| port: 3128, | ||
| latency: 32 | ||
| } | ||
| ``` |
Fixed alerts
Deprecated
MaintenanceThe maintainer of the package marked it as deprecated. This could indicate that a single version should not be used, or that the package is no longer maintained and any new vulnerabilities will not be fixed.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by9.48%
20157
- Number of package files
- increased by25%
5
- Lines of code
- increased by27.46%
181
- Number of medium maintenance alerts
- decreased by-100%
0
- Number of lines in readme file
- increased by8.22%
79
No dependency changes