read-webfile - npm Package Compare versions

Comparing version 0.1.0 to 0.2.0

doc/bugs.md

+## trigger res.on('data') after res.('end)
+state: fixed in v0.2.0
+
+### descriptor
+1. excute wf.createReadStream(`https://nodejs.prg/en/`)
+2. after httpIncommingMessage.on('end')
+3. still trigger httpIncommingMessage.on('data') once, and chunk is a part of html file
+4. can not reproduce it again when I directly use htts.get() outside wf.createReadStream
+5. cause an error: push data to readableStream after rs.push(null)
+
+### how to fix
+I add a logic: `if (rs._readableState.ended === true) return null` when res.on('data')
+
+but I still don't know fix this bug directly.

doc/README.md

+# doc

lib/createReadStreamFromWed.js

+const fs = require('fs')
+const stream = require('stream')
+const url = require('url')
+const getHttpOrHttps = require('./util/getHttpOrHttps')
+
+module.exports = function readFileFromWeb(fileURL, options) {
+  
+  // fileURL can be string or URL object
+  if(typeof fileURL === 'object') fileURL = fileURL.href 
+  if (!fileURL) throw("Need a file url to download")
+
+  // get encoding
+  var encoding = null
+  if(typeof options === 'string') encoding = options
+  if(typeof options === 'object') encoding = options.encoding
+
+  if (url.parse(fileURL).protocol === null) {
+    fileURL = 'http://' + fileURL
+  }
+  var request = getHttpOrHttps(fileURL)
+
+  const rs = new stream.Readable()
+  rs._read = function() {
+    request.get(fileURL, function(res) {
+      if (res.statusCode !== 200) {
+        throw new Error(`can not get source: ${res.statusMessage}`)
+      }
+      var isEnd = false
+      
+      res.on('data', (chunk) => {
+        if (rs._readableState.ended === true) return null
+
+        if(encoding) {
+          rs.push(chunk.toString(encoding))
+        } else {
+          rs.push(chunk)
+        }
+      })
+      res.on("end", function(){
+        rs.push(null)
+      })
+    }).on('error', function(err) {
+      throw err
+    })
+  }
+  return rs
+}

test/createReadStreamFromWeb.js

+const assert = require('assert')
+const createReadStreamFromWeb = require('../lib').createReadStreamFromWeb
+const url = require('url');
+
+var noProtocolUrl = 'music.bitfish.xyz/'
+var httpUrl = 'http://music.bitfish.xyz/'
+var httpsUrl = 'https://nodejs.org/en/'
+var redirectURL = 'https://nodejs.org/'
+
+describe('createReadStreamFromWeb', () => {
+  // this.timeout(5000);
+  describe('one arg (url)', () => {
+    describe('url is a string', () => {
+      it(`should fetch http protocol url such as ${httpUrl}`, (done) => {
+        const rs = createReadStreamFromWeb(httpUrl)
+        rs.on('data', chunk => {
+          assert(Buffer.isBuffer(chunk))
+        })
+        rs.on('end', () => {
+          done()
+        })
+      })
+      it(`should fetch https protocol url such as ${httpsUrl}`, (done) => {
+        const rs = createReadStreamFromWeb(httpsUrl)
+        rs.on('data', chunk => {
+          assert(Buffer.isBuffer(chunk))
+        })
+        rs.on('end', () => {
+          done()
+        })
+      })
+      it(`should take ${noProtocolUrl} as http protocol`, (done) => {
+        const rs = createReadStreamFromWeb(noProtocolUrl)
+        rs.on('data', chunk => {
+          assert(Buffer.isBuffer(chunk))
+        })
+        rs.on('end', () => {
+          done()
+        })
+      })
+      it(`should redirect when http code is 3XX`, (done) => {
+          done()
+      })
+    })
+    describe('url is an URL object', () => {
+      it(`should support url is an URL object`, (done) => {
+        var httpUrlObj = url.parse(httpUrl)
+        const rs = createReadStreamFromWeb(httpUrlObj)
+        rs.on('data', chunk => {
+          assert(Buffer.isBuffer(chunk))
+        })
+        rs.on('end', () => {
+          done()
+        })
+      })
+    })
+  })
+  describe('twe args (url, options)', () => {
+    describe('options is a string', () => {
+      it(`should take this string as encoding such as 'utf8'`, (done) => {
+        const rs = createReadStreamFromWeb(httpUrl, 'utf8')
+        rs.on('data', chunk => {
+          assert(Buffer.isBuffer(chunk))
+        })
+        rs.on('end', () => {
+          done()
+        })
+      })
+    })
+    describe('options is an object', () => {
+      it(`should read encoding from obj.encoding`, (done) => {
+        const rs = createReadStreamFromWeb(httpUrl, {encoding: 'utf8'})
+        rs.on('data', chunk => {
+          assert(Buffer.isBuffer(chunk))
+        })
+        rs.on('end', () => {
+          done()
+        })
+      })
+    })
+  })
+})

lib/index.js

 const readFileFromWeb = require('./readFileFromWeb')
+const createReadStreamFromWeb = require('./createReadStreamFromWed')
 
 module.exports = {
-  readFileFromWeb
+  readFileFromWeb,
+  createReadStreamFromWeb
 }

package.json

 {
   "name": "read-webfile",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "let you readFile from web just like fs.readFile",
@@ -15,3 +15,3 @@ "main": "./lib/index.js",
   "scripts": {
-    "test": "mocha ./test/"
+    "test": "mocha ./test/ --timeout 5000"
   },
@@ -18,0 +18,0 @@ "author": "",

README.md

@@ -1,3 +0,8 @@
-# web-file
+# read-webfile
+read a file from a url just like from a file path! 
 
+## Installation
+```
+npm i web-file --save
+```
 ## wf.readFileFromWeb(path[, options], callback)
@@ -7,7 +12,7 @@ basicly, this function is design to follow native API of [fs.readFile](https://nodejs.org/dist/latest-v10.x/docs/api/fs.html#fs_fs_readfile_path_options_callback).
 - path `<string>` | `<URL>` required
-  - can be a string such as https://nodejs.org
+  - can be a string such as `https://nodejs.org`
   - can be a URL object such as `new URL('https://nodejs.org')`
-  - is required, only support http or https portocol
+  - is required, and only support http or https portocol
 
-- options <Object> | <string>
+- options `<Object>` | `<string>`
   - can be a string which represent encoding such as 'utf8'
@@ -20,10 +25,6 @@ - can be object such as `{encoding: 'utf8'}`
 
-```
-npm i web-file --save
-```
 ``` js
 const wf = require('web-file')
+let url = 'https://nodejs.org/en/'
 
-var url = 'https://nodejs.org'
-
 wf.readFileFromWeb(url, 'utf8', (err, data) => {
@@ -35,5 +36,21 @@ if(err) console.error(err)
 
-## todo
-- handler 3xx redirect
+## wf.createReadStream(path[, options])
+basicly, this function is design to follow native API of [fs.createReadStream](https://nodejs.org/dist/latest-v10.x/docs/api/fs.html#fs_fs_createreadstream_path_options).
 
-- createReadStreamFromWeb
+- path `<string>` | `<URL>`
+- options `<string>` | `<Object>`
+  - encoding `<string>` Default: null
+- Returns: a Readable Stream.
+
+``` js
+const wf = require('web-file')
+let url = 'https://nodejs.org/en/'
+
+const rs = wf.createReadStreamFromWeb(url)
+rs.on('data', chunk => {
+  console.log(chunk.toString('utf8'))
+})
+rs1.on('end', () => {
+  console.log('onend')
+})
+```

test/readFileFromWeb.js

 const assert = require('assert')
 const readFileFromWeb = require('../lib').readFileFromWeb
+const url = require('url');
 
-var url = 'music.bitfish.xyz/'
+var noProtocolUrl = 'music.bitfish.xyz/'
 var httpUrl = 'http://music.bitfish.xyz/'
@@ -10,3 +11,3 @@ var httpsUrl = 'https://nodejs.org/en/'
 describe('readFileFromWeb', () => {
-  this.timeout(5000);
+  // this.timeout(5000);
   describe('two args (url, callback)', () => {
@@ -32,4 +33,4 @@ describe('url is a string', () => {
       })
-      it(`should take ${url} as http protocol`, (done) => {
-        readFileFromWeb(url, (err, data) => {
+      it(`should take ${noProtocolUrl} as http protocol`, (done) => {
+        readFileFromWeb(noProtocolUrl, (err, data) => {
           assert.equal(err, null, 
@@ -48,3 +49,4 @@ 'should not have error')
       it(`should support url is an URL object`, (done) => {
-        readFileFromWeb(httpsUrl, (err, data) => {
+        var httpUrlObj = url.parse(httpUrl)
+        readFileFromWeb(httpUrlObj, (err, data) => {
           assert.equal(err, null, 
@@ -51,0 +53,0 @@ 'should not have error')

No alert changes

Improved metrics

Total package byte prevSize

10673

92.03%

Number of package files

10

66.67%

Lines of code

252

93.85%

Number of lines in readme file

54

45.95%

Worsened metrics

Number of low supply chain risk alerts

4

33.33%

No dependency changes