cycletls - npm Package Compare versions

Comparing version 1.0.22 to 1.0.23

dist/index.d.ts

@@ -0,1 +1,2 @@
+import FormData from 'form-data';
 export interface Cookie {
@@ -27,3 +28,3 @@ name: string;
     };
-    body?: string;
+    body?: string | URLSearchParams | FormData;
     ja3?: string;
@@ -35,2 +36,4 @@ userAgent?: string;
     headerOrder?: string[];
+    insecureSkipVerify?: boolean;
+    forceHTTP1?: boolean;
 }
@@ -45,2 +48,3 @@ export interface CycleTLSResponse {
     };
+    finalUrl: string;
 }
@@ -47,0 +51,0 @@ export interface CycleTLSClient {

dist/index.js

@@ -36,2 +36,7 @@ "use strict";
 const util_1 = __importDefault(require("util"));
+const form_data_1 = __importDefault(require("form-data"));
+const stream_1 = require("stream");
+const util_2 = require("util");
+const stream_2 = __importDefault(require("stream"));
+const pipeline = (0, util_2.promisify)(stream_2.default.pipeline);
 let child;
@@ -45,3 +50,3 @@ let lastRequestID;
         if (child) {
-            new Promise((resolve, reject) => {
+            await new Promise((resolve, reject) => {
                 (0, child_process_1.exec)("taskkill /pid " + child.pid + " /T /F", (error, stdout, stderr) => {
@@ -51,4 +56,3 @@ if (error) {
                     }
-                    if (exit)
-                        process.exit();
+                    resolve(stdout);
                 });
@@ -60,4 +64,4 @@ });
         if (child) {
-            //linux/darwin os
-            new Promise((resolve, reject) => {
+            // For Linux/Darwin OS
+            await new Promise((resolve, reject) => {
                 process.kill(-child.pid);
@@ -95,2 +99,13 @@ if (exit)
 };
+// Function to convert a stream into a string
+async function streamToString(stream) {
+    const chunks = [];
+    await pipeline(stream, new stream_1.Writable({
+        write(chunk, encoding, callback) {
+            chunks.push(chunk);
+            callback();
+        }
+    }));
+    return Buffer.concat(chunks).toString('utf8');
+}
 class Golang extends events_1.EventEmitter {
@@ -174,4 +189,13 @@ server;
     }
-    request(requestId, options) {
+    async request(requestId, options) {
         lastRequestID = requestId;
+        // Check if options.body is URLSearchParams and convert to string
+        if (options.body instanceof URLSearchParams) {
+            options.body = options.body.toString();
+        }
+        // Check if options.body is FormData and convert to string
+        if (options.body instanceof form_data_1.default) {
+            options.headers = { ...options.headers, ...options.body.getHeaders() };
+            options.body = await streamToString(options.body);
+        }
         if (this.server) {
@@ -282,2 +306,6 @@ this.server.send(JSON.stringify({ requestId, options }), (err) => {
                             options.proxy = "";
+                        if (!options?.insecureSkipVerify)
+                            options.insecureSkipVerify = false;
+                        if (!options?.forceHTTP1)
+                            options.forceHTTP1 = false;
                         //convert simple cookies
@@ -309,3 +337,3 @@ const cookies = options?.cookies;
                             catch (e) { }
-                            const { Status: status, Body: body, Headers: headers } = response;
+                            const { Status: status, Body: body, Headers: headers, FinalUrl: finalUrl } = response;
                             if (headers["Set-Cookie"])
@@ -317,2 +345,3 @@ headers["Set-Cookie"] = headers["Set-Cookie"].split("/,/");
                                 headers,
+                                finalUrl,
                             });
@@ -319,0 +348,0 @@ });

package.json

 {
   "name": "cycletls",
-  "version": "1.0.22",
+  "version": "1.0.23",
   "description": "Spoof TLS/JA3 fingerprint in JS with help from Go",
@@ -12,12 +12,12 @@ "main": "dist/index.js",
   "scripts": {
-    "build:go:freebsd:amd64": " cross-env GO111MODULE=off GOOS=freebsd  GOARCH=amd64  go build -o ./dist/index-freebsd  ./src && chmod +x ./dist/index-freebsd",
-    "build:go:linux:amd64": "   cross-env GO111MODULE=off GOOS=linux    GOARCH=amd64  go build -o ./dist/index          ./src && chmod +x ./dist/index",
-    "build:go:linux:arm": "     cross-env GO111MODULE=off GOOS=linux    GOARCH=arm    go build -o ./dist/index-arm      ./src && chmod +x ./dist/index-arm",
-    "build:go:linux:arm64": "   cross-env GO111MODULE=off GOOS=linux    GOARCH=arm64  go build -o ./dist/index-arm64    ./src && chmod +x ./dist/index-arm64",
-    "build:go:mac:amd64": "     cross-env GO111MODULE=off GOOS=darwin   GOARCH=amd64  go build -o ./dist/index-mac      ./src && chmod +x ./dist/index-mac",
-    "build:go:mac:arm64": "     cross-env GO111MODULE=off GOOS=darwin   GOARCH=arm64  go build -o ./dist/index-mac-arm64 ./src && chmod +x ./dist/index-mac-arm64",
-    "build:go:windows:amd64": " cross-env GO111MODULE=off GOOS=windows  GOARCH=amd64  go build -o ./dist/index.exe      ./src",
+    "build:go:freebsd:amd64": "cd src &&  cross-env GO111MODULE=on GOOS=freebsd  GOARCH=amd64  go build -o ../dist/index-freebsd  ./ && chmod +x ../dist/index-freebsd",
+    "build:go:linux:amd64": " cd src &&   cross-env GO111MODULE=on GOOS=linux    GOARCH=amd64  go build -o ../dist/index          ./ && chmod +x ../dist/index",
+    "build:go:linux:arm": "  cd src &&    cross-env GO111MODULE=on GOOS=linux    GOARCH=arm    go build -o ../dist/index-arm      ./ && chmod +x ../dist/index-arm",
+    "build:go:linux:arm64": "cd src &&    cross-env GO111MODULE=on GOOS=linux    GOARCH=arm64  go build -o ../dist/index-arm64    ./ && chmod +x ../dist/index-arm64",
+    "build:go:mac:amd64": "   cd src &&   cross-env GO111MODULE=on GOOS=darwin   GOARCH=amd64  go build -o ../dist/index-mac      ./ && chmod +x ../dist/index-mac",
+    "build:go:mac:arm64": "    cd src &&  cross-env GO111MODULE=on GOOS=darwin   GOARCH=arm64  go build -o ../dist/index-mac-arm64 ./ && chmod +x ../dist/index-mac-arm64",
+    "build:go:windows:amd64": " cd src && cross-env GO111MODULE=on GOOS=windows  GOARCH=amd64  go build -o ../dist/index.exe      ./",
     "build:go": "concurrently npm:build:go:*",
     "build": "tsc",
-    "prebuild:go": "cross-env GO111MODULE=off go get github.com/Danny-Dasilva/CycleTLS/cycletls",
+    "prebuild:go": "cd src && cross-env GO111MODULE=on go get github.com/Danny-Dasilva/CycleTLS/cycletls",
     "prepare": "npm run build && npm run build:go",
@@ -24,0 +24,0 @@ "test": "jest"

README.md

@@ -58,3 +58,3 @@ # CycleTLS
 node ^v14.0
-golang ^v1.17x
+golang ^v1.20x
 ```
@@ -96,3 +96,3 @@
     userAgent: 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0',
-    proxy: 'http://username:[e-mail]:443'
+    proxy: 'http://username:[e-mail]:443', 
   }, 'get');
@@ -135,3 +135,26 @@
 ```
+#### Example using your own custom http.Client
 
+<details>
+
+```go
+import (
+	"github.com/Danny-Dasilva/CycleTLS/cycletls"
+	http "github.com/Danny-Dasilva/fhttp" // note this is a drop-in replacement for net/http
+)
+
+func main() {
+	ja3 := "771,52393-52392-52244-52243-49195-49199-49196-49200-49171-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-30032-11-10,29-23-24,0"
+	ua := "Chrome Version 57.0.2987.110 (64-bit) Linux"
+
+ 	 cycleClient := &http.Client{
+ 		Transport:     cycletls.NewTransport(ja3, ua),
+ 	 }
+
+	resp, err := cycleClient.Get("https://tls.peet.ws/")
+	...
+}
+```
+</details>
+
 ## Creating an instance
@@ -221,7 +244,10 @@
   // Toggle if CycleTLS should follow redirects
-  disableRedirect: true
+  disableRedirect: true,
   // Custom header order to send with request (This value will overwrite default header order)
-  headerOrder: ["cache-control", "connection", "host"]
+  headerOrder: ["cache-control", "connection", "host"],
+  // Toggle if CycleTLS should skip verify certificate (If InsecureSkipVerify is true, TLS accepts any certificate presented by the server and any host name in that certificate.)
+  insecureSkipVerify: false	
+  // Forces CycleTLS to do a http1 handshake
+  forceHTTP1: false	
 }
-);
 
@@ -242,5 +268,6 @@ ```
     ...
-  }
+  },
+  // FinalUrl returned from the server (String). This field is useful when redirection is active.
+  finalUrl: "https://final.url/"	
 }
-);
 
@@ -322,4 +349,85 @@ ```
 
+## Multiple Requests Example for Golang
 
+The general expectation for golang packages is to expect the user to implement a worker pool or any other form of goroutine/asynchronous processing. This package includes a built in Queue method that leverages a worker pool/channels for long running asynchronous requests against a set of urls.
 
+```go
+package main
+
+import (
+	"log"
+
+	cycletls "github.com/Danny-Dasilva/CycleTLS/cycletls"
+)
+
+// Static variables
+var (
+	ja3       = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0"
+	userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36"
+)
+
+// RequestConfig holds the configuration for each request.
+type RequestConfig struct {
+	URL     string
+	Method  string
+	Options cycletls.Options
+}
+
+func main() {
+	client := cycletls.Init(true) // Initialize with worker pool
+
+	// Define the requests
+	requests := []RequestConfig{
+		{
+			URL:    "http://httpbin.org/delay/4",
+			Method: "GET",
+			Options: cycletls.Options{
+				Ja3:       ja3,
+				UserAgent: userAgent,
+			},
+		},
+		{
+			URL:    "http://httpbin.org/post",
+			Method: "POST",
+			Options: cycletls.Options{
+				Body:      `{"field":"POST-VAL"}`,
+				Ja3:       ja3,
+				UserAgent: userAgent,
+			},
+		},
+		{
+			URL:    "http://httpbin.org/cookies",
+			Method: "GET",
+			Options: cycletls.Options{
+				Ja3:       ja3,
+				UserAgent: userAgent,
+				Cookies: []cycletls.Cookie{
+					{
+						Name:  "example1",
+						Value: "aaaaaaa",
+					},
+				},
+			},
+		},
+	}
+
+	// Queue the requests
+	for _, req := range requests {
+		client.Queue(req.URL, req.Options, req.Method)
+	}
+
+	// Asynchronously read responses as soon as they are available
+	// They will return as soon as they are processed
+	// e.g. Delay 3 will be returned last
+	for i := 0; i < len(requests); i++ {
+		response := <-client.RespChan
+		log.Println("Response:", response)
+	}
+
+	// Close the client
+	client.Close()
+}
+
+```
+
 # Dev Setup
@@ -498,2 +606,5 @@
 
+
+### CookieJar in JS
+
 ```js
@@ -555,7 +666,290 @@ const initCycleTLS = require("cycletls");
 
-**Golang example coming soon** 
+### CookieJar in Golang
 
+```go
+package main
+
+import (
+	"github.com/Danny-Dasilva/CycleTLS/cycletls"
+	"log"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+)
+
+func main() {
+	client := cycletls.Init()
+	jar, err := cookiejar.New(nil)
+    if err != nil {
+        log.Fatal(err)
+    }
+	// First request to set cookie
+	firstResponse, err := client.Do("https://httpbin.org/cookies/set?a=1&b=2&c=3", cycletls.Options{
+		Body: "",
+		Ja3:       "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
+		UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
+		DisableRedirect: true,
+	},
+		 "GET")
+	if err != nil {
+		log.Fatal(err)
+	}
+	firstURL, _ := url.Parse(firstResponse.FinalUrl)
+    jar.SetCookies( firstURL, firstResponse.Cookies)
+
+
+	// Second request to verify cookies, including the cookies from the first response
+	secondResponse, err := client.Do("https://httpbin.org/cookies", cycletls.Options{
+		Body: "",
+		Ja3:       "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
+		UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
+	    Headers: map[string]string{
+	        "Cookie": getHeadersFromJar(jar, firstURL),
+	    },
+	}, "GET")
+	if err != nil {
+	    log.Fatal(err)
+	}
+
+	log.Println("Second Response body:", secondResponse.Body)
+}
+
+
+func getHeadersFromJar(jar *cookiejar.Jar, url *url.URL) string {
+    cookies := jar.Cookies(url)
+    var cookieStrs []string
+    for _, cookie := range cookies {
+        cookieStrs = append(cookieStrs, cookie.Name+"="+cookie.Value)
+    }
+    return strings.Join(cookieStrs, "; ")
+}
+
+```
+
 </details>
 
+### How do I send multipart/form-data in CycleTLS
 
+<details>
+
+### Javascript Text form-data
+```js
+const initCycleTLS = require("cycletls");
+const FormData = require('form-data');
+
+(async () => {
+  const cycleTLS = await initCycleTLS();
+
+  const formData = new FormData();
+  formData.append("key1", "value1");
+  formData.append("key2", "value2");
+  
+  const response = await cycleTLS('http://httpbin.org/post', {
+      body: formData,
+      headers: {
+          'Content-Type': 'multipart/form-data',
+      },
+  }, 'post');
+
+  console.log(response);
+
+  cycleTLS.exit();
+})();
+
+```
+
+
+### Javascript File form-data
+```js
+const initCycleTLS = require("cycletls");
+const FormData = require('form-data');
+const fs = require('fs');
+
+(async () => {
+  const cycleTLS = await initCycleTLS();
+
+  const formData = new FormData();
+  const fileStream = fs.createReadStream("../go.mod");
+  formData.append('file', fileStream);
+
+  
+  const response = await cycleTLS('http://httpbin.org/post', {
+      body: formData,
+      headers: {
+          'Content-Type': 'multipart/form-data',
+      },
+  }, 'post');
+
+  console.log(response);
+
+  cycleTLS.exit();
+})();
+
+```
+
+### Golang Text form-data
+```golang
+package main
+
+import (
+	"bytes"
+	"github.com/Danny-Dasilva/CycleTLS/cycletls"
+	"log"
+	"mime/multipart"
+)
+
+func main() {
+	client := cycletls.Init()
+
+	// Prepare a buffer to write our multipart form
+	var requestBody bytes.Buffer
+	multipartWriter := multipart.NewWriter(&requestBody)
+
+	// Add form fields
+	multipartWriter.WriteField("key1", "value1")
+	multipartWriter.WriteField("key2", "value2")
+
+	contentType := multipartWriter.FormDataContentType()
+	// Close the writer before making the request
+	multipartWriter.Close()
+
+	response, err := client.Do("http://httpbin.org/post", cycletls.Options{
+		Body: requestBody.String(),
+		Headers: map[string]string{
+			"Content-Type": contentType,
+		},
+	}, "POST")
+
+	if err != nil {
+		log.Print("Request Failed: " + err.Error())
+	}
+
+	log.Println(response.Body)
+}
+```
+
+
+### Golang file upload form-data
+```golang
+package main
+
+import (
+	"github.com/Danny-Dasilva/CycleTLS/cycletls"
+	"bytes"
+	"io"
+	"log"
+	"mime/multipart"
+	"os"
+)
+
+func main() {
+  client := cycletls.Init()
+
+  // Prepare a buffer to write our multipart form
+  var requestBody bytes.Buffer
+  multipartWriter := multipart.NewWriter(&requestBody)
+
+  // Add a file
+  fileWriter, err := multipartWriter.CreateFormFile("fieldname", "filename")
+  if err != nil {
+      log.Fatal("CreateFormFile Error: ", err)
+  }
+
+  // Open the file that you want to upload
+  file, err := os.Open("path/to/your/file")
+  if err != nil {
+      log.Fatal("File Open Error: ", err)
+  }
+  defer file.Close()
+
+  // Copy the file to the multipart writer
+  _, err = io.Copy(fileWriter, file)
+  if err != nil {
+      log.Fatal("File Copy Error: ", err)
+  }
+
+  // Close the writer before making the request
+  contentType := multipartWriter.FormDataContentType()
+  multipartWriter.Close()
+
+  response, err := client.Do("http://httpbin.org/post", cycletls.Options{
+      Body: requestBody.String(),
+      Headers: map[string]string{
+          "Content-Type": contentType,
+      },
+  }, "POST")
+
+  if err != nil {
+      log.Print("Request Failed: " + err.Error())
+  }
+
+  log.Println(response.Body)
+}
+```
+
+
+If requested encoding helpers can be added to the repo for golang 
+</details>
+
+### How do I send a application/x-www-form-urlencoded Post request
+
+<details>
+
+### Javascript application/x-www-form-urlencoded form
+```js
+const initCycleTLS = require("cycletls");
+(async () => {
+  const cycleTLS = await initCycleTLS();
+
+  const urlEncodedData = new URLSearchParams();
+  urlEncodedData.append('key1', 'value1');
+  urlEncodedData.append('key2', 'value2');
+
+  const response = await cycleTLS('http://httpbin.org/post', {
+      body: urlEncodedData,
+      headers: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+      },
+  }, 'post');
+
+  console.log(response);
+
+  cycleTLS.exit();
+})();
+
+```
+### Golang application/x-www-form-urlencoded form
+```golang
+package main
+
+import (
+    "log"
+	  "net/url"
+    "github.com/Danny-Dasilva/CycleTLS/cycletls"
+)
+
+func main() {
+
+	client := cycletls.Init()
+
+	// Prepare form data
+	form := url.Values{}
+	form.Add("key1", "value1")
+	form.Add("key2", "value2")
+
+	response, err := client.Do("http://httpbin.org/post", cycletls.Options{
+		Body: form.Encode(),
+		Headers: map[string]string{
+			"Content-Type": "application/x-www-form-urlencoded",
+		},
+	}, "POST")
+	if err != nil {
+		log.Print("Request Failed: " + err.Error())
+	}
+	log.Println(response.Body)
+}
+
+```
+</details>
+
 ### How do I download images?
@@ -562,0 +956,0 @@

No alert changes

Improved metrics

Total package byte prevSize

80673366

increased by16.38%

Lines of code

452

increased by7.88%

Number of lines in readme file

1079

increased by57.52%

No dependency changes