file-combination-tool - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1

cli.js

 #!/usr/bin/env node
 
 const path = require("path");
+const fs = require("fs").promises;
+const os = require("os");
 const { readConfigFile } = require("./config");
 const { combineFiles } = require("./fileProcessor");
+const yargs = require("yargs");
 
+const defaultExtensions = [".js", ".ts", ".jsx", ".tsx", ".py", ".rb", ".java", ".c", ".cpp", ".cs", ".php", ".go", ".rs", ".swift", ".kt", ".md"];
+
+const defaultLicenseHeaders = [
+    // Pattern to match one or more consecutive license headers
+    /^(\/\*[\s\S]*?(\*\/|(?:license|copyright|spdx|contrib|opensearch|elasticsearch)[\s\S]*?\*\/)[\s\*]*)+/gi
+];
+
+// ANSI escape codes for coloring
+const colors = {
+    red: "\x1b[31m",
+    green: "\x1b[32m",
+    yellow: "\x1b[33m",
+    blue: "\x1b[34m",
+    magenta: "\x1b[35m",
+    cyan: "\x1b[36m",
+    reset: "\x1b[0m"
+};
+
+function getOpenFolderCommand(relativeFilePath) {
+    const folderPath = path.dirname(relativeFilePath);
+    const platform = os.platform();
+
+    let command;
+
+    if (platform === 'win32') {
+        // Windows
+        command = `start ${folderPath}`;
+    } else if (platform === 'darwin') {
+        // macOS
+        command = `open ${folderPath}`;
+    } else {
+        // Other (Linux and others)
+        command = `xdg-open ${folderPath}`;
+    }
+
+    return command;
+}
+
+function colorizeCommand(command, color) {
+    return `${color}${command}${colors.reset}`;
+}
+
 async function main() {
-    const args = process.argv.slice(2);
+    const argv = yargs
+        .option('baseDir', {
+            alias: 'b',
+            describe: 'Base directory to process files from',
+            type: 'string',
+            default: process.cwd()
+        })
+        .option('include', {
+            alias: 'i',
+            describe: 'Files or folders to include (comma-separated)',
+            type: 'string'
+        })
+        .option('exclude', {
+            alias: 'e',
+            describe: 'Patterns to exclude (comma-separated)',
+            type: 'string',
+            default: 'node_modules/,venv/,.git/'
+        })
+        .option('extensions', {
+            alias: 'x',
+            describe: 'File extensions to process (comma-separated)',
+            type: 'string',
+            default: defaultExtensions.join(',')
+        })
+        .option('output', {
+            alias: 'o',
+            describe: 'Output file name',
+            type: 'string'
+        })
+        .option('config', {
+            alias: 'c',
+            describe: 'Path to config file',
+            type: 'string'
+        })
+        .help()
+        .argv;
 
-    if (args.length < 1) {
-        console.error("Please provide the path to the configuration file.");
-        process.exit(1);
+    let config;
+
+    if (argv.config) {
+        config = await readConfigFile(argv.config);
+    } else {
+        config = {
+            baseDir: argv.baseDir,
+            include: argv.include ? argv.include.split(',') : [],
+            exclude: argv.exclude ? argv.exclude.split(',') : ['node_modules/', 'venv/', '.git/'],
+            extensions: argv.extensions ? argv.extensions.split(',') : defaultExtensions,
+            output: argv.output || `combined_${path.basename(argv.baseDir)}.txt`,
+            licenseHeaders: defaultLicenseHeaders
+        };
     }
 
-    const configPath = args[0];
-    const config = await readConfigFile(configPath);
-
-    const baseDir = path.resolve(config.baseDir || ".");
+    const baseDir = path.resolve(config.baseDir);
     const includePaths = config.include || [];
     const skipPatterns = config.exclude || [];
-    const outputPath =
-        config.output ||
-        path.join(process.cwd(), `combined_${path.basename(baseDir)}.txt`);
-    const headerPatterns = config.headerPatterns || [
-        "/\\*[\\s\\S]*?Copyright[\\s\\S]*?\\*/",
-        "/\\*\\s*\\*\\s*Licensed[\\s\\S]*?under the License\\.\\s*\\*/",
-    ];
+    const extensions = config.extensions || defaultExtensions;
+    const outputPath = config.output ? path.resolve(config.output) : path.join(process.cwd(), `combined_${path.basename(baseDir)}.txt`);
+    const licenseHeaders = config.licenseHeaders || defaultLicenseHeaders;
 
@@ -34,6 +117,13 @@ await combineFiles(
         skipPatterns,
-        headerPatterns
+        licenseHeaders,
+        extensions
     );
+
+    // Output command to open the folder containing the output file
+    const command = getOpenFolderCommand(outputPath);
+    const colorizedCommand = colorizeCommand(command, colors.green);
+    console.log(`\nTo open the folder containing the output file, run the following command:`);
+    console.log(colorizedCommand);
 }
 
 main().catch(console.error);

fileProcessor.js

 const fs = require("fs").promises;
 const path = require("path");
-const os = require("os");
-const { fileExists, shouldSkipFile, removeHeaders, countTokens } = require("./utils");
+const { fileExists, shouldSkipFile, removeLicenseHeaders, countTokens } = require("./utils");
 const { outputLargestFiles, outputLargestFolders } = require("./outputFormatters");
 
-// ANSI escape codes for coloring
-const colors = {
-  red: "\x1b[31m",
-  green: "\x1b[32m",
-  yellow: "\x1b[33m",
-  blue: "\x1b[34m",
-  magenta: "\x1b[35m",
-  cyan: "\x1b[36m",
-  reset: "\x1b[0m"
-};
-
-function getOpenFolderCommand(relativeFilePath) {
-  const folderPath = path.dirname(relativeFilePath);
-  const platform = os.platform();
-
-  let command;
-
-  if (platform === 'win32') {
-    // Windows
-    command = `start ${folderPath}`;
-  } else if (platform === 'darwin') {
-    // macOS
-    command = `open ${folderPath}`;
-  } else {
-    // Other (Linux and others)
-    command = `xdg-open ${folderPath}`;
-  }
-
-  return command;
-}
-
-function colorizeCommand(command, color) {
-  return `${color}${command}${colors.reset}`;
-}
-
-async function combineFiles(baseDir, includePaths, outputPath, skipPatterns, headerPatterns) {
+async function combineFiles(baseDir, includePaths, outputPath, skipPatterns, licenseHeaderPatterns, extensions) {
   try {
@@ -53,5 +17,5 @@ let content = "";
       if (stats.isDirectory()) {
-        const dirFiles = await getAllFiles(fullPath, baseDir, [], skipPatterns);
+        const dirFiles = await getAllFiles(fullPath, baseDir, [], skipPatterns, extensions);
         for (const filePath of dirFiles) {
-          const { fileContent, fileSize, tokenCount } = await processFile(baseDir, filePath, skipPatterns, headerPatterns);
+          const { fileContent, fileSize, tokenCount } = await processFile(baseDir, filePath, skipPatterns, licenseHeaderPatterns);
           content += fileContent;
@@ -63,4 +27,4 @@ if (fileSize !== null) {
         }
-      } else if (isIncludePath || !shouldSkipFile(currentPath, skipPatterns)) {
-        const { fileContent, fileSize, tokenCount } = await processFile(baseDir, currentPath, skipPatterns, headerPatterns);
+      } else if (isIncludePath || (!shouldSkipFile(currentPath, skipPatterns) && extensions.includes(path.extname(currentPath)))) {
+        const { fileContent, fileSize, tokenCount } = await processFile(baseDir, currentPath, skipPatterns, licenseHeaderPatterns);
         content += fileContent;
@@ -90,8 +54,2 @@ if (fileSize !== null) {
     console.log(`\nEstimated total number of tokens: ${totalTokenCount}`);
-
-    // Output command to open the folder containing the output file using the cli without changing the current directory
-    const command = getOpenFolderCommand(outputPath);
-    const colorizedCommand = colorizeCommand(command, colors.green); // Using green color for the command
-    console.log(`\nTo open the folder containing the output file, run the following command:`);
-    console.log(colorizedCommand);
   } catch (error) {
@@ -102,3 +60,3 @@ console.error("Error:", error);
 
-async function processFile(baseDir, filePath, skipPatterns, headerPatterns) {
+async function processFile(baseDir, filePath, skipPatterns, licenseHeaderPatterns) {
   const fullPath = path.join(baseDir, filePath);
@@ -110,3 +68,3 @@ let fileContent = "";
     fileContent = await fs.readFile(fullPath, "utf8");
-    fileContent = removeHeaders(fileContent, headerPatterns);
+    fileContent = removeLicenseHeaders(fileContent, licenseHeaderPatterns);
     tokenCount = countTokens(fileContent);
@@ -123,13 +81,3 @@ fileContent = `\n//===== FILE: ${filePath} =====//\n\n${fileContent.trim()}\n`;
 
-function updateFolderSizes(folderSizes, filePath, fileSize, tokenCount) {
-  const rootFolder = filePath.split(path.sep)[0];
-  if (!folderSizes.has(rootFolder)) {
-    folderSizes.set(rootFolder, { size: 0, tokens: 0 });
-  }
-  const folderStats = folderSizes.get(rootFolder);
-  folderStats.size += fileSize;
-  folderStats.tokens += tokenCount;
-}
-
-async function getAllFiles(dir, baseDir, fileList = [], skipPatterns) {
+async function getAllFiles(dir, baseDir, fileList = [], skipPatterns, extensions) {
   const files = await fs.readdir(dir);
@@ -142,6 +90,6 @@ for (const file of files) {
       if (!shouldSkipFile(relativePath, skipPatterns)) {
-        await getAllFiles(filePath, baseDir, fileList, skipPatterns);
+        await getAllFiles(filePath, baseDir, fileList, skipPatterns, extensions);
       }
     } else if (
-      [".js", ".ts", ".jsx", ".tsx", ".md"].includes(path.extname(file)) &&
+      extensions.includes(path.extname(file)) &&
       !shouldSkipFile(relativePath, skipPatterns)
@@ -155,2 +103,12 @@ ) {
 
-module.exports = { combineFiles };
+function updateFolderSizes(folderSizes, filePath, fileSize, tokenCount) {
+  const rootFolder = filePath.split(path.sep)[0];
+  if (!folderSizes.has(rootFolder)) {
+    folderSizes.set(rootFolder, { size: 0, tokens: 0 });
+  }
+  const folderStats = folderSizes.get(rootFolder);
+  folderStats.size += fileSize;
+  folderStats.tokens += tokenCount;
+}
+
+module.exports = { combineFiles };

package.json

 {
   "name": "file-combination-tool",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "A tool to combine multiple files from a specific directory",
@@ -28,3 +28,6 @@ "main": "index.js",
   },
-  "homepage": "https://github.com/ashwin-pc/merge_files#readme"
+  "homepage": "https://github.com/ashwin-pc/merge_files#readme",
+  "dependencies": {
+    "yargs": "^17.7.2"
+  }
 }

README.md

@@ -1,3 +0,1 @@
-Updated README.md for npx usage
-
 # File Combination Tool
@@ -7,3 +5,3 @@
 
-The File Combination Tool is a Node.js script that combines multiple files from a specified directory (and its subdirectories) into a single output file. It provides options to include or exclude specific files or directories, remove header comments, and output statistics about the largest files and folders processed.
+The File Combination Tool is a Node.js script that combines multiple files from a specified directory (and its subdirectories) into a single output file. It supports various programming languages and file types, providing options to include or exclude specific files or directories, remove header comments, and output statistics about the largest files and folders processed.
 
@@ -13,2 +11,3 @@ ## Features
 - Combine multiple files into a single output file
+- Support for multiple programming languages and file types
 - Include specific files/folders or process all files in the base directory
@@ -30,3 +29,3 @@ - Exclude files/folders based on patterns
 ```
-npx file-combination-tool path/to/your/config.json
+npx file-combination-tool [options]
 ```
@@ -45,3 +44,3 @@
 ```
-file-combination-tool path/to/your/config.json
+file-combination-tool [options]
 ```
@@ -51,35 +50,76 @@
 
-Run the script using one of the following commands:
+You can use the tool with minimal configuration, or customize it with command-line arguments or a config file.
 
-Using npx:
+### Simplest Usage
+
+To combine all supported files in the current directory:
+
 ```
-npx file-combination-tool path/to/your/config.json
+npx file-combination-tool
 ```
 
-If installed globally:
+This will create a file named `combined_[current-directory-name].txt` in the current directory.
+
+### Command-line Arguments
+
 ```
-file-combination-tool path/to/your/config.json
+npx file-combination-tool [options]
 ```
 
-Where `path/to/your/config.json` is the path to your configuration file.
+Options (all are optional):
+- `--baseDir`, `-b`: Base directory to process files from (default: current directory)
+- `--include`, `-i`: Files or folders to include (comma-separated)
+- `--exclude`, `-e`: Patterns to exclude (comma-separated, default: 'node_modules/,venv/,.git/')
+- `--extensions`, `-x`: File extensions to process (comma-separated, default: .js,.ts,.jsx,.tsx,.py,.rb,.java,.c,.cpp,.cs,.php,.go,.rs,.swift,.kt,.md)
+- `--output`, `-o`: Output file name (default: combined_[baseDir-name].txt)
+- `--config`, `-c`: Path to config file (if using a config file instead of command-line args)
 
-Or locally
+Examples:
 
+1. Combine Python files:
+   ```
+   npx file-combination-tool --extensions .py
+   ```
+
+2. Combine Java and Kotlin files:
+   ```
+   npx file-combination-tool --extensions .java,.kt
+   ```
+
+3. Exclude additional patterns:
+   ```
+   npx file-combination-tool --exclude "test/,*.spec.js"
+   ```
+
+4. Specify output file:
+   ```
+   npx file-combination-tool --output combined-project.txt
+   ```
+
+5. Combine files from a different directory:
+   ```
+   npx file-combination-tool --baseDir ../another-project
+   ```
+
+### Config File
+
+For more complex scenarios, you can use a config file:
+
 ```
-node cli.js path/to/your/config.json
+npx file-combination-tool --config path/to/your/config.json
 ```
 
-## Configuration
+Config file structure:
 
-Create a JSON configuration file with the following structure:
-
 ```json
 {
   "baseDir": "/path/to/your/base/directory",
-  "include": ["folder1", "folder2/file.js"],
-  "exclude": ["\\.test\\.(js|ts|jsx|tsx)$", "\\.d\\.ts$", "fixtures/"],
+  "include": ["folder1", "folder2/file.py"],
+  "exclude": ["test/", "*.spec.py"],
+  "extensions": [".py", ".pyx"],
   "output": "combined_output.txt",
   "headerPatterns": [
-    "/\\*[\\s\\S]*?Copyright[\\s\\S]*?\\*/",
-    "/\\*\\s*\\*\\s*Licensed[\\s\\S]*?under the License\\.\\s*\\*/"
+    "#.*",
+    "'''[\\s\\S]*?'''",
+    '"""[\\s\\S]*?"""'
   ]
@@ -89,10 +129,44 @@ }
 
-### Configuration Options
+## Supported Languages
 
-- `baseDir` (required): The base directory to process files from.
-- `include` (optional): An array of files or folders to include. If omitted, all files in the base directory will be processed.
-- `exclude` (optional): An array of regex patterns for files or folders to exclude.
-- `output` (optional): The name of the output file. Defaults to `combined_[baseDir].txt` in the current working directory.
-- `headerPatterns` (optional): An array of regex patterns for headers to remove from the files.
+The tool supports the following file extensions by default:
+.js, .ts, .jsx, .tsx, .py, .rb, .java, .c, .cpp, .cs, .php, .go, .rs, .swift, .kt, .md
 
+You can specify additional file extensions using the `--extensions` option or in the config file.
+
+## Local Development
+
+If you've cloned this repository and want to run the tool locally:
+
+1. Navigate to the project directory:
+   ```
+   cd path/to/file-combination-tool
+   ```
+
+2. Install dependencies:
+   ```
+   npm install
+   ```
+
+3. Run the tool using one of these methods:
+
+   a. Using node directly:
+      ```
+      node cli.js
+      ```
+
+   b. Using npm scripts:
+      ```
+      npm run combine
+      ```
+
+4. (Optional) Create a symlink for global-like usage:
+   ```
+   npm link
+   ```
+   After linking, you can use the tool from any directory:
+   ```
+   file-combination-tool
+   ```
+
 ## Output
@@ -108,38 +182,6 @@
 
-## Example
-
-1. Create a configuration file named `config.json`:
-
-```json
-{
-  "baseDir": "../OpenSearch-Dashboards/src/plugins",
-  "include": ["data", "vis_builder"],
-  "exclude": [
-    "\\.test\\.(js|ts|jsx|tsx)$",
-    "\\.d\\.ts$",
-    "fixtures/",
-    "mock",
-    "api\\.md",
-    "_generated_/"
-  ],
-  "output": "combined_plugins.txt",
-  "headerPatterns": [
-    "/\\*[\\s\\S]*?Copyright[\\s\\S]*?\\*/",
-    "/\\*\\s*\\*\\s*Licensed[\\s\\S]*?under the License\\.\\s*\\*/"
-  ]
-}
-```
-
-2. Run the script:
-
-```
-node index.js config.json
-```
-
-This will process the specified plugins, combine their files (excluding test files, type definitions, fixtures, mocks, API docs, and generated files), remove the specified headers, and output the result to `combined_plugins.txt`.
-
 ## Limitations
 
-- The tool currently only processes files with the following extensions: .js, .ts, .jsx, .tsx, and .md.
 - The token counting is a simple estimate and may not match exactly with more sophisticated tokenization methods.
+- Header removal patterns are predefined for common languages. For other languages, you may need to specify custom patterns.
 
@@ -152,2 +194,2 @@ ## Contributing
 
-MIT License
+MIT License

utils.js

@@ -16,7 +16,4 @@ const fs = require("fs").promises;
 
-function removeHeaders(content, headerPatterns) {
-  return headerPatterns.reduce(
-    (acc, pattern) => acc.replace(new RegExp(pattern, 'g'), ""),
-    content
-  );
+function removeLicenseHeaders(content, licenseHeaders) {
+  return licenseHeaders.reduce((acc, pattern) => acc.replace(pattern, ''), content).trim();
 }
@@ -32,4 +29,4 @@
   shouldSkipFile,
-  removeHeaders,
+  removeLicenseHeaders,
   countTokens
-};
+};

No alert changes

Improved metrics

Total package byte prevSize

16832

increased by19.72%

Lines of code

310

increased by15.24%

Number of lines in readme file

187

increased by28.08%

Worsened metrics

Dependency count

1

increased byInfinity%

Number of low supply chain risk alerts

4

increased by33.33%

Dependency changes

• + Addedyargs@^17.7.2

• + Addedansi-regex@5.0.1(transitive)

• + Addedansi-styles@4.3.0(transitive)

• + Addedcliui@8.0.1(transitive)

• + Addedcolor-convert@2.0.1(transitive)

• + Addedcolor-name@1.1.4(transitive)

• + Addedemoji-regex@8.0.0(transitive)

• + Addedescalade@3.2.0(transitive)

• + Addedget-caller-file@2.0.5(transitive)

• + Addedis-fullwidth-code-point@3.0.0(transitive)

• + Addedrequire-directory@2.1.1(transitive)

• + Addedstring-width@4.2.3(transitive)

• + Addedstrip-ansi@6.0.1(transitive)

• + Addedwrap-ansi@7.0.0(transitive)

• + Addedy18n@5.0.8(transitive)

• + Addedyargs@17.7.2(transitive)

• + Addedyargs-parser@21.1.1(transitive)