socks-scraper - npm Package Compare versions

Comparing version 1.0.4 to 1.1.0-h

index.js

@@ -1,28 +0,19 @@
-const { SocksClient } = require('socks');
-const { request } = require('undici');
-const { Worker } = require('node:worker_threads');
-const path = require('path')
+const { ProxyAgent } = require('proxy-agent');
+const needle = require('needle');
 
-const workerPath = path.join(__dirname, 'worker.js')
 const ipPortRegex = /^(?:\d{1,3}\.){3}\d{1,3}:\d+$/;
 
 /**
- * @typedef {import('socks/typings/common/constants').SocksProxyType} SocksScraper.SocksProxyType
+ * @typedef {'http' | 'socks5' | 'socks4'} SocksScraper.SocksProxyType
  */
 
 /**
- * @typedef {import('undici/types/header').IncomingHttpHeaders} SocksScraper.IncomingHttpHeaders
+ * @typedef {5 | 4} SocksScraper.SocksProxyOldType
  */
 
 /**
- * like { checkURL, checkURLPort, headers}
- * @typedef {Object} SocksScraper.ClassOptions
- * @property {?SocksScraper.IncomingHttpHeaders} headers
-*/
-
-/**
  * @typedef {Object} IDefaultMessage
  * @property {SocksScraper.SocksProxyType} sockType
  * @property {number} timeout
- * @property {string[]} proxyChunk 
+ * @property {string[]} proxyChunk
  * @property {number} chunkSize
@@ -40,8 +31,14 @@ */
 
+/**
+ * @param {needle.NeedleResponse} response
+ */
+function defaultProxyCallback(response) {
+	return response.body?.origin || JSON.parse(response.body)?.origin
+}
+
 class SocksScraper {
 	/**
-	 * @param {?string[]} sites 
-	 * @param {?SocksScraper.ClassOptions} options 
+	 * @param {?string[]} sites
 	 */
-	constructor(sites = [], options = { headers: null }) {
+	constructor(sites = []) {
 
@@ -55,14 +52,12 @@ /**
 		/**
-		 * @private
+		 * @public
+		 * @type {Set<string>}
 		 */
-		this.unCheckedProxies = []
+		this.unCheckedProxies = new Set();
 
 		/**
-		 * @private
-		 * @type {typeof options.headers}
+		 * @public
+		 * @type {SocksScraper.IDefaultProxy[]}
 		 */
-		this.headers = options?.headers || {
-			'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0',
-			'Connection': 'keep-alive'
-		}
+		this.checkedProxies = [];
 	}
@@ -73,3 +68,3 @@
 	 * @public
-	 * @param {string[]} sites 
+	 * @param {string[]} sites
 	 */
@@ -89,5 +84,19 @@ addSites(sites) {
 	/**
+	 * Clears the list of unchecked proxies
+	 */
+	clearUnCheckedProxies() {
+		this.unCheckedProxies.clear();
+	}
+
+	/**
+	 * Clears the list of checked proxies
+	 */
+	clearCheckedProxies() {
+		this.checkedProxies = [];
+	}
+
+	/**
 	 * at `0` the best at `-1` the worst
 	 * @public
-	 * @param {SocksScraper.IDefaultProxy[]} proxies 
+	 * @param {SocksScraper.IDefaultProxy[]} proxies
 	 * @returns {SocksScraper.IDefaultProxy[]}
@@ -102,14 +111,17 @@ */
 	 * @public
-	 * @param {string} url 
-	 * @returns {Promise<string[] | []>}
+	 * @param {string} url
 	 */
 	async getProxiesFromRawSite(url) {
 		try {
-			const response = await request(url, { method: "GET", headers: this.headers })
-			const responseText = await response.body.text()
+			const res = await needle('get', url, {
+				response_timeout: 10000,
+				follow_max: 5,
+				rejectUnauthorized: false
+			})
 
-			const proxyList = responseText.split('\n').filter((x) => x.match(ipPortRegex))
-			return proxyList
+			if (!res.body || typeof res.body !== 'string') return [];
+
+			return res.body.split(/\r|\n|<br>/).filter((a) => a !== '').filter((x) => x.match(ipPortRegex)) || [];
 		} catch (error) {
-			return null
+			return [];
 		}
@@ -119,27 +131,55 @@ }
 	/**
+	 * @param {SocksScraper.SocksProxyOldType} type
+	 * @param {string} address
+	 * @param {number?} timeout
+	 * @param {string?} website
+	 * @param {number?} checkURLPort
+	 */
+	static async checkSocksProxy(type, address, timeout = undefined, website = undefined, checkURLPort = undefined) {
+		const newType = type === 5 ? 'socks5' : 'socks4';
+
+		return await SocksScraper.isAliveProxy(newType, address, timeout, website) || null;
+	}
+
+	/**
 	 * Check ip:port to see if it is a proxy and if it works at all
 	 * @public
-	 * @param {SocksScraper.SocksProxyType} type 
-	 * @param {string} address 
-	 * @param {number} timeout 
-	 * @returns {Promise<?SocksScraper.IDefaultProxy>}
+	 * @param {SocksScraper.SocksProxyType} type
+	 * @param {string} address
+	 * @param {number?} timeout
 	 */
-	static async checkSocksProxy(type, address, timeout = 5000, checkURL = 'http://ip-api.com/ip', checkURLPort = 80) {
-		const [host, portStr] = address.split(':');
-		const port = Number(portStr);
+	static async isAliveProxy(type, address, timeout = 6000, website = 'https://httpbin.org/ip', callback = defaultProxyCallback) {
+		try {
 
-		const startTime = performance.now();
+			const agent = new ProxyAgent({
+				getProxyForUrl: () => `${type}://${address}`,
+				timeout,
+				rejectUnauthorized: false
+			});
 
-		try {
-			await SocksClient.createConnection({
-				timeout,
-				command: 'connect',
-				destination: { host: checkURL, port: checkURLPort },
-				proxy: { host, port, type }
+			const startTime = performance.now();
+			const response = await needle('get', website, {
+				agent: agent,
+				follow: 10,
+				open_timeout: 10000,
+				response_timeout: 10000,
+				read_timeout: 5000,
+				rejectUnauthorized: false
 			})
 			const latency = Math.round(performance.now() - startTime)
 
-			return { address, host, port, latency }
+			if (callback && callback(response)) {
+				const [host, portStr] = address.split(':');
+
+				return {
+					address,
+					host,
+					port: Number(portStr),
+					latency
+				}
+			}
+
+			return false
 		} catch (error) {
-			return null
+			return false
 		}
@@ -150,3 +190,2 @@ }
 	 * @public
-	 * @returns {Promise<void>}
 	 */
@@ -157,3 +196,3 @@ async updateUncheckedProxies() {
 
-		this.unCheckedProxies = Array.prototype.concat(...notTestedProxyList)
+		this.unCheckedProxies = new Set(Array.prototype.concat(...notTestedProxyList))
 	}
@@ -164,33 +203,20 @@
 	 * @public
-	 * @param {SocksScraper.SocksProxyType} sockType 
-	 * @param {number} timeout 
-	 * @param {number} [chunkSize=10000] number of Promise.all treatments for 1 worker
-	 * @param {number} [workerCount=1] count of workers
-	 * @returns {Promise<?SocksScraper.IDefaultProxy[]>}
+	 * @param {SocksScraper.SocksProxyType} sockType
+	 * @param {number} timeout
+	 * @returns {Promise<SocksScraper.IDefaultProxy[]>}
 	 */
-	async getWorkedSocksProxies(sockType, timeout, workerCount = 1, chunkSize = 10000) {
-		const workers = [], workedProxyLists = [];
-		const proxyChunkSize = Math.ceil(this.unCheckedProxies.length / workerCount);
+	async getWorkedSocksProxies(sockType, timeout) {
+		this.clearCheckedProxies()
 
-		for (let i = 0; i < workerCount; i++) {
-			const worker = new Worker(workerPath);
+		const checkedProxiesPromise = Array.from(this.unCheckedProxies).map(async (a) => SocksScraper.isAliveProxy(sockType, a, timeout))
 
-			worker.on('message', (workedProxyList) => {
-				workedProxyLists.push(...workedProxyList);
-			});
+		const checkedProxies = await Promise.all(checkedProxiesPromise)
 
-			const proxyChunk = this.unCheckedProxies.slice(i * proxyChunkSize, (i + 1) * proxyChunkSize);
-			worker.postMessage({ sockType, timeout, proxyChunk, chunkSize });
-			workers.push(worker);
-		}
+		for (const proxy of checkedProxies) {
+			if (!proxy) continue
 
-		await Promise.all(workers.map((worker) => new Promise((resolve) => {
-			worker.once('message', resolve);
-		})));
-
-		for (const worker of workers) {
-			worker.terminate();
+			this.checkedProxies.push(proxy)
 		}
 
-		return workedProxyLists;
+		return this.checkedProxies
 	}
@@ -197,0 +223,0 @@ }

package.json

 {
 	"dependencies": {
-		"socks": "^2.8.1",
-		"undici": "^6.10.1"
+		"needle": "^3.3.1",
+		"proxy-agent": "^6.4.0"
 	},
 	"name": "socks-scraper",
-	"version": "1.0.4",
+	"version": "1.1.0h",
 	"description": "Uses your references to obtain and verify proxies ",
 	"main": "index.js",
-	"devDependencies": {},
 	"scripts": {
@@ -12,0 +11,0 @@ "test": "node example.js"

README.md

@@ -12,4 +12,4 @@ # Proxy Scraper
 # Lib Dependencies
-- undici
-- socks
+- needle
+- proxy-agent
 
@@ -34,2 +34,3 @@ # JSDoc
 const socksScraper = new SocksScraper([
+	"https://api.proxyscrape.com/?request=displayproxies&status=alive",
 	"https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5",
@@ -39,3 +40,7 @@ "https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4",
 	"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt",
-	"https://shieldcommunity.net/sockets.txt"
+	'https://api.proxyscrape.com/?request=displayproxies&status=alive&proxytype=socks4',
+	'https://api.proxyscrape.com/?request=displayproxies&status=alive&proxytype=socks5',
+	'https://openproxylist.xyz/socks4.txt',
+	'https://openproxylist.xyz/socks5.txt',
+	'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt'
 ])
@@ -53,22 +58,31 @@
 
-console.log('Done updating unchecked proxies!');
+console.log(`Done updating unchecked proxies! (${socksScraper.unCheckedProxies.size})`);
 
 // Get a list of proxies from all sites, check if they work and return the best ones
-const wsp4 = await socksScraper.getWorkedSocksProxies(4, timeout)
+const wsp4 = await socksScraper.getWorkedSocksProxies('socks4', timeout)
+
 // Sort the list by latency and take the fastest proxy
 const bestWSP4 = SocksScraper.filterByLatency(wsp4)[0]
 
-console.log(`The best socks4 proxy is ${bestWSP4.host}:${bestWSP4.port} with latency ${bestWSP4.latency}ms`)
+console.log(`The best socks4 proxy is ${bestWSP4.host}:${bestWSP4.port} with latency ${bestWSP4.latency}ms (${wsp4.length})`)
 
-const wsp5 = await socksScraper.getWorkedSocksProxies(5, timeout, 1, 20000)
+const wsp5 = await socksScraper.getWorkedSocksProxies('socks5', timeout)
 const bestWSP5 = SocksScraper.filterByLatency(wsp5)[0]
 
-console.log(`The best socks5 proxy is ${bestWSP5.host}:${bestWSP5.port} with latency ${bestWSP5.latency}ms`)
+console.log(`The best socks5 proxy is ${bestWSP5.host}:${bestWSP5.port} with latency ${bestWSP5.latency}ms (${wsp5.length})`)
 
-// Check my socks5 proxy to see if it works at all
-const mySocks5Proxy = await SocksScraper.checkSocksProxy(5, '94.131.14.66:1080', 4000)
-const isAlive = Boolean(mySocks5Proxy)
+/* only if you have VERY good internet...
 
-console.log(`My socks5 proxy is ${isAlive ? 'alive' : 'dead'}`)
-console.log(mySocks5Proxy)
+const http = await socksScraper.getWorkedSocksProxies('http', timeout)
+const bestHttp = SocksScraper.filterByLatency(http)[0]
+
+console.log(`The best http proxy is ${bestHttp.host}:${bestHttp.port} with latency ${bestHttp.latency}ms`)
+*/
+
+//  Check my socks5 proxy to see if it works at all
+const mySocks4Proxy = await SocksScraper.isAliveProxy('socks4', '3.10.93.50:80', 10000)
+const isAlive = Boolean(mySocks4Proxy)
+
+console.log(`My socks4 proxy is ${isAlive ? 'alive' : 'dead'}`)
+console.log(mySocks4Proxy);
 ```
@@ -75,0 +89,0 @@ ```js

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Manifest confusion

Supply chain risk

This package has inconsistent metadata. This could be malicious or caused by an error when publishing the package.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

24118

increased by19.65%

Number of package files

8

increased by60%

Lines of code

233

increased by28.73%

Number of lines in readme file

93

increased by17.72%

Worsened metrics

Number of low quality alerts

1

increased byInfinity%

Number of medium supply chain risk alerts

2

increased byInfinity%

Dependency changes

• + Addedneedle@^3.3.1

• + Addedproxy-agent@^6.4.0

• + Added@tootallnate/quickjs-emscripten@0.23.0(transitive)

• + Addedagent-base@7.1.1(transitive)

• + Addedast-types@0.13.4(transitive)

• + Addedbasic-ftp@5.0.5(transitive)

• + Addeddata-uri-to-buffer@6.0.2(transitive)

• + Addeddebug@4.3.7(transitive)

• + Addeddegenerator@5.0.1(transitive)

• + Addedescodegen@2.1.0(transitive)

• + Addedesprima@4.0.1(transitive)

• + Addedestraverse@5.3.0(transitive)

• + Addedesutils@2.0.3(transitive)

• + Addedfs-extra@11.2.0(transitive)

• + Addedget-uri@6.0.3(transitive)

• + Addedgraceful-fs@4.2.11(transitive)

• + Addedhttp-proxy-agent@7.0.2(transitive)

• + Addedhttps-proxy-agent@7.0.5(transitive)

• + Addediconv-lite@0.6.3(transitive)

• + Addedjsonfile@6.1.0(transitive)

• + Addedlru-cache@7.18.3(transitive)

• + Addedms@2.1.3(transitive)

• + Addedneedle@3.3.1(transitive)

• + Addednetmask@2.0.2(transitive)

• + Addedpac-proxy-agent@7.0.2(transitive)

• + Addedpac-resolver@7.0.1(transitive)

• + Addedproxy-agent@6.4.0(transitive)

• + Addedproxy-from-env@1.1.0(transitive)

• + Addedsafer-buffer@2.1.2(transitive)

• + Addedsax@1.4.1(transitive)

• + Addedsocks-proxy-agent@8.0.4(transitive)

• + Addedsource-map@0.6.1(transitive)

• + Addedtslib@2.8.1(transitive)

• + Addeduniversalify@2.0.1(transitive)

• - Removedsocks@^2.8.1

• - Removedundici@^6.10.1

• - Removedundici@6.21.0(transitive)