detect-secrets-js - npm Package Compare versions

Comparing version 2.0.0 to 2.0.1

package.json

 {
   "name": "detect-secrets-js",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "A JavaScript implementation of Yelp's detect-secrets tool - no Python required",
@@ -5,0 +5,0 @@ "main": "wasm-version/dist/index.js",

README.md

@@ -16,2 +16,3 @@ # detect-secrets-js
 - **Compatible API**: Similar interface to Yelp's detect-secrets for easy migration
+- **Memory Efficient**: Automatically skips binary files and handles large codebases
 
@@ -43,2 +44,8 @@ ## Installation
 detect-secrets-js --output results.json
+
+# Enable file size limits to prevent memory issues with very large files
+detect-secrets-js --limit-file-size
+
+# Set a custom maximum file size (in KB) when limits are enabled
+detect-secrets-js --limit-file-size --max-file-size 2048
 ```
@@ -59,3 +66,5 @@
     excludeDirs: ['node_modules', 'dist'],
-    checkMissed: true
+    checkMissed: true,
+    limitFileSize: false,  // Set to true to enable file size limits
+    maxFileSize: 2 * 1024 * 1024  // Custom max file size in bytes (2MB) when limits are enabled
   });
@@ -89,2 +98,4 @@
 | `output` | `-o, --output <file>` | Output file path |
+| `limitFileSize` | `-l, --limit-file-size` | Enable file size limits to prevent memory issues |
+| `maxFileSize` | `--max-file-size <size>` | Maximum file size to scan in KB (default: no limit) |
 
@@ -97,2 +108,11 @@ ## How It Works
 
+### Memory Management
+
+By default, the tool will scan all files regardless of size, but you can enable memory protection features:
+
+1. **Binary File Detection**: Automatically skips binary files like images, executables, and compressed files
+2. **Optional Size Limits**: Use `--limit-file-size` to enable file size limits
+3. **Custom Size Limits**: Set your own maximum file size with `--max-file-size`
+4. **Automatic Truncation**: Very large text files can be truncated to prevent memory issues
+
 ## Types of Secrets Detected
@@ -117,2 +137,3 @@
 4. **Similar Detection Patterns**: Implements the same secret detection patterns
+5. **Memory Efficient**: Better handling of large repositories and binary files
 
@@ -126,2 +147,3 @@ ## Version History
 - Improved performance and cross-platform compatibility
+- Added memory-efficient handling of large repositories
 
@@ -128,0 +150,0 @@ ## License

wasm-version/bin/cli.js

@@ -23,3 +23,5 @@ #!/usr/bin/env node
   .option('-v, --verbose', 'Include additional information')
-  .option('-o, --output <file>', 'Output file path');
+  .option('-o, --output <file>', 'Output file path')
+  .option('-l, --limit-file-size', 'Enable file size limits to prevent memory issues')
+  .option('--max-file-size <size>', 'Maximum file size to scan in KB (default: no limit)', parseInt);
 
@@ -32,3 +34,3 @@ program.parse(process.argv);
 function formatResults(results) {
-  const { secrets, missed_secrets } = results;
+  const { secrets, missed_secrets, truncated } = results;
   
@@ -41,2 +43,8 @@ if (secrets.length === 0 && missed_secrets.length === 0) {
   
+  // Note if any files were truncated
+  if (truncated) {
+    output += chalk.yellow('Note: Some files were truncated due to size limits.\n');
+    output += chalk.yellow('Use --max-file-size to increase the limit or remove --limit-file-size to scan without limits.\n\n');
+  }
+  
   // Group secrets by file
@@ -117,3 +125,5 @@ const fileGroups = {};
       checkMissed: options.checkMissed || false,
-      verbose: options.verbose || false
+      verbose: options.verbose || false,
+      limitFileSize: options.limitFileSize || false,
+      maxFileSize: options.maxFileSize ? options.maxFileSize * 1024 : undefined
     };
@@ -120,0 +130,0 @@

wasm-version/package.json

 {
   "name": "detect-secrets-js",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "A JavaScript implementation of Yelp's detect-secrets tool - no Python required",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

wasm-version/src/index.ts

@@ -17,2 +17,55 @@ import { loadPyodide } from 'pyodide';
 
+// Constants
+const DEFAULT_MAX_FILE_SIZE = 0; // No default file size limit (0 means no limit)
+const BINARY_FILE_EXTENSIONS = [
+  '.pack', '.gz', '.zip', '.jar', '.war', '.ear', '.class', '.so', '.dll', '.exe', 
+  '.obj', '.o', '.a', '.lib', '.pyc', '.pyo', '.jpg', '.jpeg', '.png', '.gif', 
+  '.bmp', '.ico', '.tif', '.tiff', '.mp3', '.mp4', '.avi', '.mov', '.wmv', '.flv',
+  '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx'
+];
+
+// Helper function to check if a file is likely binary
+function isLikelyBinaryFile(filePath: string, fileSize: number): boolean {
+  // Check file extension
+  const ext = path.extname(filePath).toLowerCase();
+  if (BINARY_FILE_EXTENSIONS.includes(ext)) {
+    return true;
+  }
+  
+  // Check if it's in a binary-like directory
+  if (filePath.includes('/.git/') || 
+      filePath.includes('/node_modules/') || 
+      filePath.includes('/__pycache__/') ||
+      filePath.includes('/.next/cache/')) {
+    return true;
+  }
+  
+  // Try to read a small chunk to detect binary content
+  try {
+    const fd = fs.openSync(filePath, 'r');
+    const buffer = Buffer.alloc(Math.min(4096, fileSize));
+    fs.readSync(fd, buffer, 0, buffer.length, 0);
+    fs.closeSync(fd);
+    
+    // Check for null bytes which often indicate binary data
+    for (let i = 0; i < buffer.length; i++) {
+      if (buffer[i] === 0) {
+        return true;
+      }
+    }
+    
+    // Try to decode as UTF-8 - if it fails, likely binary
+    try {
+      buffer.toString('utf8');
+    } catch (e) {
+      return true;
+    }
+  } catch (e) {
+    // If we can't read the file, assume it's not binary
+    return false;
+  }
+  
+  return false;
+}
+
 /**
@@ -34,3 +87,3 @@ * Initialize the WebAssembly module and Python environment
     
-    // Load Pyodide - use the default CDN path instead of local files
+    // Load Pyodide - use the default CDN path
     console.log('Loading Pyodide...');
@@ -85,12 +138,56 @@ pyodideInstance = await loadPyodide();
   try {
+    // Get the max file size from options
+    const maxFileSize = options.maxFileSize || DEFAULT_MAX_FILE_SIZE;
+    
+    // If max file size is set and content is too large, truncate it to prevent memory issues
+    let truncated = false;
+    if (maxFileSize > 0 && content.length > maxFileSize) {
+      content = content.substring(0, maxFileSize);
+      truncated = true;
+      console.warn(`File ${filePath} is too large, scanning only the first ${maxFileSize} bytes`);
+    }
+    
     // Convert options to a Python-compatible format
-    const checkMissed = options.checkMissed ? 'True' : 'False';
+    const checkMissed = options.checkMissed ? true : false;
     
+    // Set up Python variables
+    pyodideInstance.globals.set('js_file_content', content);
+    pyodideInstance.globals.set('js_file_path', filePath);
+    pyodideInstance.globals.set('js_check_missed', checkMissed);
+    
     // Call the Python scan_file function
-    const resultJson = await pyodideInstance.runPythonAsync(`
-      scan_file(${JSON.stringify(content)}, ${JSON.stringify(filePath)}, ${checkMissed})
+    await pyodideInstance.runPythonAsync(`
+      import json
+      try:
+          result_json = scan_file(js_file_content, js_file_path, js_check_missed)
+          js_result = result_json
+      except Exception as e:
+          import traceback
+          error_msg = traceback.format_exc()
+          print(f"Python error: {str(e)}\\n{error_msg}")
+          js_result = json.dumps({"error": str(e), "secrets": [], "missed_secrets": []})
     `);
     
+    // Get the result from Python
+    const resultJson = pyodideInstance.globals.get('js_result');
+    
+    // Check if we got a valid result
+    if (!resultJson) {
+      throw new Error('No result returned from Python scanner');
+    }
+    
     // Parse the results
-    return JSON.parse(resultJson);
+    const results = JSON.parse(resultJson);
+    
+    // Check if there was an error
+    if (results.error) {
+      throw new Error(`Python error: ${results.error}`);
+    }
+    
+    // Add a note if the file was truncated
+    if (truncated) {
+      results.truncated = true;
+    }
+    
+    return results;
   } catch (error: unknown) {
@@ -114,8 +211,31 @@ console.error('Error scanning content:', error);
   try {
+    // Get file stats
+    const stats = fs.statSync(filePath);
+    
+    // Skip directories
+    if (stats.isDirectory()) {
+      return { secrets: [], missed_secrets: [] };
+    }
+    
+    // Get the max file size from options
+    const maxFileSize = options.maxFileSize || DEFAULT_MAX_FILE_SIZE;
+    
+    // Check if it's a binary file
+    if (isLikelyBinaryFile(filePath, stats.size)) {
+      console.log(`Skipping likely binary file: ${filePath}`);
+      return { secrets: [], missed_secrets: [] };
+    }
+    
+    // Skip large files if a limit is set and limitFileSize option is true
+    if (maxFileSize > 0 && stats.size > maxFileSize && options.limitFileSize) {
+      console.log(`Skipping large file (${Math.round(stats.size / 1024)}KB): ${filePath}`);
+      return { secrets: [], missed_secrets: [] };
+    }
+    
+    // Read and scan the file
     const content = fs.readFileSync(filePath, 'utf-8');
     return scanContent(content, filePath, options);
   } catch (error: unknown) {
-    console.error(`Error reading or scanning file ${filePath}:`, error);
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    throw new Error(`Failed to scan file ${filePath}: ${errorMessage}`);
+    console.warn(`Skipping file ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
+    return { secrets: [], missed_secrets: [] };
   }
@@ -143,29 +263,54 @@ }
 
+  // Default excluded directories if none provided
+  const defaultExcludeDirs = [
+    'node_modules', '.git', 'dist', 'build', 'coverage', '.next',
+    '__pycache__', '.venv', 'venv', 'env', '.env'
+  ];
+  
+  // Default excluded file patterns if none provided
+  const defaultExcludeFiles = [
+    '*.min.js', '*.min.css', '*.map', '*.lock', '*.svg', '*.woff', '*.ttf', '*.eot',
+    '*.jpg', '*.jpeg', '*.png', '*.gif', '*.ico', '*.pdf', '*.zip', '*.tar.gz'
+  ];
+
   // Get all files in the directory
   const getFiles = (dir: string, excludeDirs: string[] = []): string[] => {
     let files: string[] = [];
-    const entries = fs.readdirSync(dir, { withFileTypes: true });
     
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
+    try {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
       
-      // Skip excluded directories
-      if (entry.isDirectory()) {
-        if (excludeDirs.some(pattern => 
-          new RegExp(pattern).test(entry.name) || 
-          entry.name === 'node_modules' || 
-          entry.name === '.git'
-        )) {
-          continue;
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        
+        // Skip excluded directories
+        if (entry.isDirectory()) {
+          const excludePatterns = [...defaultExcludeDirs, ...(options.excludeDirs || [])];
+          if (excludePatterns.some(pattern => 
+            new RegExp(`^${pattern.replace(/\*/g, '.*')}$`).test(entry.name) || 
+            entry.name === pattern
+          )) {
+            continue;
+          }
+          
+          try {
+            files = files.concat(getFiles(fullPath, excludeDirs));
+          } catch (err) {
+            console.warn(`Skipping directory ${fullPath}: ${err instanceof Error ? err.message : String(err)}`);
+          }
+        } else {
+          // Skip excluded files
+          const excludePatterns = [...defaultExcludeFiles, ...(options.excludeFiles || [])];
+          if (excludePatterns.some(pattern => {
+            const regex = new RegExp(`^${pattern.replace(/\./g, '\\.').replace(/\*/g, '.*')}$`);
+            return regex.test(entry.name);
+          })) {
+            continue;
+          }
+          
+          files.push(fullPath);
         }
-        files = files.concat(getFiles(fullPath, excludeDirs));
-      } else {
-        // Skip excluded files
-        if (options.excludeFiles && options.excludeFiles.some(pattern => 
-          new RegExp(pattern).test(entry.name)
-        )) {
-          continue;
-        }
-        files.push(fullPath);
       }
+    } catch (err) {
+      console.warn(`Error reading directory ${dir}: ${err instanceof Error ? err.message : String(err)}`);
     }
@@ -178,6 +323,15 @@
   
-  // Scan each file
+  // Track if any files were truncated
+  let anyTruncated = false;
+  
+  // Scan each file with error handling for individual files
   for (const file of files) {
     try {
       const fileResults = await scanFile(file, options);
+      
+      // Check if this file was truncated
+      if (fileResults.truncated) {
+        anyTruncated = true;
+      }
+      
       results.secrets = results.secrets.concat(fileResults.secrets);
@@ -189,2 +343,7 @@ results.missed_secrets = results.missed_secrets.concat(fileResults.missed_secrets);
   }
+  
+  // Set the truncated flag if any files were truncated
+  if (anyTruncated) {
+    results.truncated = true;
+  }
 
@@ -191,0 +350,0 @@ return results;

wasm-version/src/types.ts

@@ -39,2 +39,12 @@ /**
   output?: string;
+  
+  /**
+   * Enable file size limits (default: false)
+   */
+  limitFileSize?: boolean;
+  
+  /**
+   * Maximum file size to scan in bytes (default: 0, no limit)
+   */
+  maxFileSize?: number;
 }
@@ -105,2 +115,7 @@
   missed_secrets: MissedSecret[];
+  
+  /**
+   * Whether the file was truncated due to size limits
+   */
+  truncated?: boolean;
 } 

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

131512

6.33%

Lines of code

3326

5.05%

Number of lines in readme file

152

16.92%

Number of medium supply chain risk alerts

1

-50%

No dependency changes