postlight2md - npm Package Compare versions

Comparing version 0.2.0 to 0.3.0

CHANGELOG.md

@@ -5,10 +5,31 @@ # Changelog
 
+## [0.3.0]
+
+### Added
+
+- Added support for processing URLs from a file with specified concurrency.
+- Added `-u, --url-file` option to specify a file containing URLs to process.
+- Added `-c, --concurrency` option to set the number of concurrent requests.
+
+### Changed
+
+- Updated the `main` function to handle URL files and concurrency.
+- Improved error handling and logging.
+
+### Fixed
+
+- Fixed issue with output file name generation when using the `-o` option.
+
+## [0.2.0]
+
+### Added
+
+- Output option to specify filename for saved content.
+
 ## [0.1.0]
+
 ### Added
+
 - Initial implementation of URL parser with command-line options.
 - README with installation instructions, usage, and options for `postlight2md`.
 - Version badge to README.
-
-## [0.2.0]
-### Added
-- Output option to specify filename for saved content.

index.js

@@ -8,6 +8,7 @@ #!/usr/bin/env node
 import { hideBin } from 'yargs/helpers';
+import Bluebird from 'bluebird';
+import ProgressBar from 'progress';
 
 const argv = yargs(hideBin(process.argv))
   .usage('Usage: $0 <url> [options]')
-  .demandCommand(1, 'You need to provide a URL to parse')
   .option('format', {
@@ -44,2 +45,21 @@ alias: 'f',
     coerce: (arg) => (arg === '' ? true : arg),
+  })
+  .option('url-file', {
+    alias: 'u',
+    describe: 'File containing URLs to process',
+    type: 'string',
+  })
+  .option('concurrency', {
+    alias: 'c',
+    describe: 'Number of concurrent requests',
+    type: 'number',
+    default: 1,
+  })
+  .check((argv) => {
+    if (!argv._.length && !argv.urlFile) {
+      throw new Error(
+        'You need to provide a URL to parse or a file containing URLs'
+      );
+    }
+    return true;
   }).argv;
@@ -83,8 +103,9 @@
 
-Parser.parse(url, options)
-  .then((result) => {
+async function processUrl(url, options, bar) {
+  try {
+    const result = await Parser.parse(url, options);
     const content = result.content;
-    if (argv.output) {
+    if (argv.output !== false) {
       let filename;
-      if (argv.output === true) {
+      if (argv.output === true || argv.urlFile) {
         const title = result.title || 'output';
@@ -104,6 +125,33 @@ filename =
     }
-  })
-  .catch((error) => {
+  } catch (error) {
     console.error(error);
     process.exitCode = 1;
+  } finally {
+    if (bar) bar.tick(); // Update the progress bar
+  }
+}
+
+async function processUrlsFromFile(filePath, concurrency, options) {
+  const fileContent = fs.readFileSync(filePath, 'utf-8');
+  const urls = fileContent.split(/\r?\n/).filter(Boolean);
+  const bar = new ProgressBar('Processing [:bar] :current/:total', {
+    total: urls.length,
+  }); // Add this line
+  await Bluebird.map(urls, (url) => processUrl(url, options, bar), {
+    concurrency,
   });
+}
+
+async function main() {
+  if (argv.urlFile) {
+    argv.output = argv.output !== false ? true : false;
+    await processUrlsFromFile(argv.urlFile, argv.concurrency, argv);
+  } else {
+    await processUrl(argv._[0], argv);
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package.json

 {
   "name": "postlight2md",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "main": "index.js",
@@ -15,4 +15,5 @@ "type": "module",
     "@postlight/parser": "^2.2.3",
+    "progress": "^2.0.3",
     "yargs": "^17.7.2"
   }
 }

README.md

@@ -31,2 +31,4 @@ # postlight2md
 - `-o, --output [filename]`: Specify the output file name. If not provided, the title of the content will be used to generate the file name.
+- `-u, --url-file <file>`: Specify a file containing URLs to process, one per line.
+- `-c, --concurrency <number>`: Number of concurrent requests. Default is `1`.
 
@@ -77,2 +79,20 @@ ### Examples
 
+Process URLs from a file with default concurrency:
+
+```sh
+postlight2md -u urls.txt
+```
+
+Process URLs from a file with specified concurrency:
+
+```sh
+postlight2md -u urls.txt -c 5
+```
+
+Note: When using the `-u` option, the `-o` option defaults to generating filenames based on the title of each URL's content. To output to the console instead, explicitly set `-o` to `false`:
+
+```sh
+postlight2md -u urls.txt -o false
+```
+
 ## License
@@ -79,0 +99,0 @@

No alert changes

Improved metrics

Total package byte prevSize

7659

44.67%

Lines of code

141

46.88%

Number of lines in readme file

103

24.1%

Worsened metrics

Dependency count

3

50%

Dependency changes

• + Added

  progress@^2.0.3

• + Added

  progress@2.0.3
  (transitive)