custom-rss - npm Package Compare versions

Comparing version 0.3.0 to 0.3.1

app.js

 var app = require('connect')();
 var createRepostGuard = require('./src/repost-guard');
+var feeds = require('./config').feeds;
 var fs = require('fs');
@@ -18,3 +19,3 @@ var log = require('./src/util').log;
 
-require('./config').feeds.forEach(function(feed) {
+feeds.forEach(function(feed) {
   var middleware = require(path.join(__dirname, 'src/feeds', feed.name));
@@ -21,0 +22,0 @@ app.use('/'+ feed.name, middleware.bind(null, feed));

config.json

@@ -11,3 +11,3 @@ {
           "tokens": [
-            "Get a job", "Sponsored", "Pokémon"
+            "Get a job", "Sponsored", "Angry Birds", "Pokémon", "PS4", "Xbox"
           ]
@@ -20,3 +20,3 @@ }
 
-      "hnappQuery": "score>1%20%7C%20comments>1%20-type%3Acomment%20-type%3Aask%20-type%3Ashow%20-type%3Ajob%20-host%3Aqz.com%20-host%3Ayahoo.com",
+      "hnappQuery": "score%3E3%20%7C%20comments%3E1%20-type%3Acomment%20-type%3Aask%20-type%3Ashow%20-type%3Ajob%20-host%3Aqz.com%20-host%3Ayahoo.com",
 
@@ -28,6 +28,6 @@ "filters": [
           "tokens": [
-            "#C", "\\.NET", "Angular", "BEM", "DuckDuckGo", "Java", "Kotlin",
+            "C#", "\\.NET", "Angular", "BEM", "DuckDuckGo", "Java", "Kotlin",
             "Lua", "MatLab", "OCaml", "Perl", "R", "Raspberry\\s?Pi", "Rust",
             "Surface Pro", "Xamarin", "Vim", "Emacs", "Bitcoin", "LaTeX",
-            "TypeScript", "C\\+\\+", "TPP"
+            "TypeScript", "C\\+\\+", "TPP", "Lisp", "Scheme", "Clojure", "F#"
           ]
@@ -39,3 +39,4 @@ },
           "tokens": [
-            "Trump", "Bernie", "Hillary", "San Bernardino", "Theranos"
+            "Trump", "Bernie", "Hillary", "San Bernardino", "Theranos", "FBI",
+            "NSA"
           ]
@@ -81,4 +82,4 @@ }
           "tokens": [
-            "Trump", "Bernie", "Hillary", "San Bernardino", "Brexit", 
-            "Sharapova", "hoverboard"
+            "Trump", "Bernie", "Sanders", "Hillary", "Clinton", "San Bernardino",
+            "Brexit", "Sharapova", "hoverboard"
           ]
@@ -85,0 +86,0 @@ }

package.json

 {
   "name": "custom-rss",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Filtering RSS because Zapier is too expensive.",
@@ -5,0 +5,0 @@ "main": "src/feeds/index.js",

src/entry-logger.js

@@ -5,2 +5,7 @@ var path = require('path');
 module.exports = function createEntryLogger(delegate) {
+  // delegate.directory: a full path
+  // delegate.feedName: a dasherized string
+  // delegate.lineLimit: a natural integer
+  // delegate.sync: a bool
+  // delegate.onReady: a function, if 'sync' is off
   return {
@@ -16,7 +21,12 @@ logEntry: function(entry) {
       var data = this.data + line;
-      var firstLineEnd = data.indexOf('\n');
-      if (this.lines === delegate.lineLimit) {
-        this.archiveBuffer += data.substring(0, firstLineEnd + 1);
-        data = data.substring(firstLineEnd);
+      var lines = this.lines + 1;
+
+      var archiveUntil;
+      if (lines > delegate.lineLimit) {
+        // End index of lines past the limit, include '\n'.
+        archiveUntil = util.nthIndexOf(data, '\n', lines - delegate.lineLimit) + 1;
+        this.archiveBuffer += data.substring(0, archiveUntil);
+        data = data.substring(archiveUntil);
       }
+
       this.setData(data);
@@ -23,0 +33,0 @@ },

src/feeds/gama-sutra.js

 var fetchFeed = require('../fetch-feed');
 var filterFeed = require('../filter-feed');
-var patterns = require('../util').patterns;
+var url = require('url');
 
@@ -10,6 +10,12 @@ function transformMeta(root) {
 module.exports = function(config, request, response) {
+  config.originalURL = 'http://feeds.feedburner.com/GamasutraFeatureArticles';
+  config.url = url.format({
+    protocol: 'http', host: request.headers.host, pathname: config.name
+  });
+
   fetchFeed({
-    url: 'http://feeds.feedburner.com/GamasutraFeatureArticles',
+    url: config.originalURL,
     onResponse: function(resFetch, data) {
       response.setHeader('Content-Type', resFetch.headers['content-type']);
+
       filterFeed({
@@ -16,0 +22,0 @@ config: config,

src/feeds/hacker-news.js

 var fetchFeed = require('../fetch-feed');
 var filterFeed = require('../filter-feed');
 var patterns = require('../util').patterns;
+var url = require('url');
 
@@ -28,6 +29,12 @@ var rScorePrefix = /^\d+\s+\S+\s+/;
 module.exports = function(config, request, response) {
+  config.originalURL = 'http://hnapp.com/rss?q='+ config.hnappQuery;
+  config.url = url.format({
+    protocol: 'http', host: request.headers.host, pathname: config.name
+  });
+
   fetchFeed({
-    url: 'http://hnapp.com/rss?q='+ config.hnappQuery,
+    url: config.originalURL,
     onResponse: function(resFetch, data) {
       response.setHeader('Content-Type', resFetch.headers['content-type']);
+
       filterFeed({
@@ -34,0 +41,0 @@ config: config,

src/feeds/nyt-business.js

 var fetchFeed = require('../fetch-feed');
 var filterFeed = require('../filter-feed');
-var patterns = require('../util').patterns;
+var url = require('url');
 
@@ -15,2 +15,7 @@ function transformLink(entry) {
 module.exports = function(config, request, response) {
+  config.originalURL = 'http://www.nytimes.com/services/xml/rss/nyt/Business.xml';
+  config.url = url.format({
+    protocol: 'http', host: request.headers.host, pathname: config.name
+  });
+
   fetchFeed({
@@ -20,2 +25,3 @@ url: 'http://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
       response.setHeader('Content-Type', resFetch.headers['content-type']);
+
       filterFeed({
@@ -22,0 +28,0 @@ config: config,

src/feeds/quartz.js

 var fetchFeed = require('../fetch-feed');
 var filterFeed = require('../filter-feed');
-var patterns = require('../util').patterns;
+var url = require('url');
 
@@ -10,6 +10,12 @@ function transformMeta(root) {
 module.exports = function(config, request, response) {
+  config.originalURL = 'http://qz.com/feed/';
+  config.url = url.format({
+    protocol: 'http', host: request.headers.host, pathname: config.name
+  });
+
   fetchFeed({
-    url: 'http://qz.com/feed/',
+    url: config.originalURL,
     onResponse: function(resFetch, data) {
       response.setHeader('Content-Type', resFetch.headers['content-type']);
+
       filterFeed({
@@ -16,0 +22,0 @@ config: config,

src/feeds/ray-wenderlich.js

 var fetchFeed = require('../fetch-feed');
 var filterFeed = require('../filter-feed');
-var patterns = require('../util').patterns;
+var url = require('url');
 
@@ -10,6 +10,12 @@ function transformMeta(root) {
 module.exports = function(config, request, response) {
+  config.originalURL = 'https://www.raywenderlich.com/feed?max-results=1';
+  config.url = url.format({
+    protocol: 'http', host: request.headers.host, pathname: config.name
+  });
+
   fetchFeed({
-    url: 'https://www.raywenderlich.com/feed?max-results=1',
+    url: config.originalURL,
     onResponse: function(resFetch, data) {
       response.setHeader('Content-Type', resFetch.headers['content-type']);
+
       filterFeed({
@@ -16,0 +22,0 @@ config: config,

src/feeds/yahoo-tech.js

 var fetchFeed = require('../fetch-feed');
 var filterFeed = require('../filter-feed');
-var patterns = require('../util').patterns;
+var url = require('url');
 
@@ -10,6 +10,12 @@ function transformMeta(root) {
 module.exports = function(config, request, response) {
+  config.originalURL = 'https://www.yahoo.com/tech/rss';
+  config.url = url.format({
+    protocol: 'http', host: request.headers.host, pathname: config.name
+  });
+
   fetchFeed({
-    url: 'https://www.yahoo.com/tech/rss',
+    url: config.originalURL,
     onResponse: function(resFetch, data) {
       response.setHeader('Content-Type', resFetch.headers['content-type']);
+
       filterFeed({
@@ -16,0 +22,0 @@ config: config,

src/fetch-feed.js

 var util = require('./util');
 
 module.exports = function fetchFeed(delegate) {
+  // delegate.url: a url string
+  // delegate.verbose: a bool
+  // delegate.onError: an error handler
+  // delegate.onResponse: a response and data handler
+
   var request = util.request(delegate.url);
@@ -5,0 +10,0 @@

src/filter-feed.js

@@ -40,3 +40,12 @@ var createEntryLogger = require('./entry-logger');
 
-function filterFeed(delegate) {
+function main(delegate) {
+  // delegate.config.filters: an array of filter objects for createFilters
+  // delegate.data: an xml string
+  // delegate.findId: a function returning id string for xml-transformer 'entry'
+  // delegate.find(Entry|Link|Title): optional functions return data for xml-transformer 'entry'
+  // delegate.guardReposts: a bool
+  // delegate.logger: a logger whose 'logEntry' takes a dictionary
+  // delegate.shouldSkipEntry: 
+  // delegate.transform(Entry|Meta): optional transform functions that mutate given xml-transformer's 'string'
+
   var filters = createFilters(delegate.config.filters);
@@ -86,6 +95,10 @@
 
-module.exports = function(delegate) {
+function filterFeed(delegate) {
   // Remove any XML stylesheets; we won't be serving them.
   delegate.data = delegate.data.replace(/<\?xml-stylesheet[^]+?\?>\s*/g, '');
 
+  // Update URL values in feed.
+  delegate.data = delegate.data.split(delegate.config.originalURL)
+    .join(delegate.config.url);
+
   // Wait for logger.
@@ -97,3 +110,3 @@ delegate.logger = createEntryLogger({
     sync: false,
-    onReady: filterFeed.bind(null, delegate)
+    onReady: main.bind(null, delegate)
   });
@@ -113,2 +126,7 @@
   delegate.logger.setUp();
-};
+}
+
+filterFeed.createFilters = createFilters;
+filterFeed.defaultShouldSkipEntry = defaultShouldSkipEntry;
+
+module.exports = filterFeed;

src/repost-guard.js

@@ -5,2 +5,7 @@ var path = require('path');
 module.exports = function createRepostGuard(delegate) {
+  // delegate.directory: a full path
+  // delegate.feedPageSize: a natural integer
+  // delegate.lineLimit: a natural integer
+  // delegate.sync: a bool
+  // delegate.onReady: a function, if 'sync' is off
   return {
@@ -75,15 +80,2 @@ checkLink: function(link) {
 
-    currentPageIndex: function() {
-      var i = delegate.feedPageSize;
-      var index;
-      while (i--) {
-        index = this.data.lastIndexOf('\n', index - 1);
-        if (index === -1) {
-          index = 0;
-          break;
-        }
-      }
-      return index;
-    },
-
     resetData: function() {
@@ -97,5 +89,7 @@ this.data = this.dataToCheck = null;
       this.data = data;
-      this.dataToCheck = data.substring(0, this.currentPageIndex());
       var matchResults = data.match(util.patterns.line);
       this.lines = matchResults ? matchResults.length : 0;
+
+      var currentPageIndex = util.nthLastIndexOf(this.data, '\n', delegate.feedPageSize);
+      this.dataToCheck = data.substring(0, (currentPageIndex === -1) ? 0 : currentPageIndex);
     },
@@ -102,0 +96,0 @@

src/util.js

@@ -5,2 +5,4 @@ var fs = require('fs');
 
+// section: debugging
+
 var mode = process.env.NODE_ENV || 'development';
@@ -37,2 +39,22 @@ module.exports.mode = mode;
 
+// section: regex
+
+module.exports.patterns = {
+  domain: /:\/\/(?:www\.)?([^\/]+)/,
+  line: /\n/g,
+  createFromTokens: function(escapedTokens) {
+    return new RegExp('\\b(' +
+      escapedTokens.join('|').replace(/\s/g, '\\s') +
+    ')\\b');
+  }
+};
+
+// section: http
+
+module.exports.normalizeLink = function(link) {
+  var parsed = url.parse(link);
+  parsed.host = parsed.host.replace(/^www\./, '');
+  return parsed.host + parsed.pathname;
+};
+
 module.exports.request = function() {
@@ -50,17 +72,4 @@ var module, protocol;
 
-module.exports.normalizeLink = function(link) {
-  var parsed = url.parse(link);
-  return parsed.host + parsed.pathname;
-};
+// section: fs
 
-module.exports.patterns = {
-  domain: /:\/\/(?:www\.)?([^\/]+)/,
-  line: /\n/g,
-  createFromTokens: function(escapedTokens) {
-    return new RegExp('\\b(' +
-      escapedTokens.join('|').replace(/\s/g, '\\s') +
-    ')\\b');
-  }
-};
-
 function handleFileError(delegate, retry, error) {
@@ -124,8 +133,31 @@ if (error.code === 'ENOENT') {
 
+// section: async
+
 module.exports.callOn = function(calls, fn) {
   var remaining = calls - 1;
   return function() {
-    if (remaining > 0) { return remaining--; }
+    if (remaining > 0) {
+      remaining -= 1;
+      return;
+    }
     fn();
   };
 };
+
+// section: string
+
+module.exports.nthIndexOf = function(string, search, n) {
+  var i;
+  for (i = 0; n > 0 && i !== -1; n -= 1) {
+    i = string.indexOf(search, /* fromIndex */ i ? (i + 1) : i);
+  }
+  return i;
+};
+
+module.exports.nthLastIndexOf = function(string, search, n) {
+  var i;
+  for (i = string.length; n > 0 && i !== -1; n -= 1) {
+    i = string.lastIndexOf(search, /* fromIndex */ i ? (i - 1) : i);
+  }
+  return i;
+};

src/xml-transformer.js

@@ -37,2 +37,4 @@ var log = require('./util').log;
 module.exports = function createXMLTransformer(delegate) {
+  // delegate.string: an xml string
+  // delegate.verbose: a bool
   return {
@@ -39,0 +41,0 @@ creator: createXMLTransformer,

tests/index.js

 require('./entry-logger-tests');
+require('./filter-feed-tests');
 require('./repost-guard-tests');
+require('./util-tests');
 require('./xml-transformer-tests');

tests/xml-transformer-tests.js

@@ -12,5 +12,5 @@ var assert = require('assert');
     '<root>\n',
-    '\t<entry rel-src="/foo.html">\n\t\t<title>foo</title>\n\t</entry>\n',
-    '\t<entry rel-src="/bar.html">\n\t\t<title>bar</title>\n\t</entry>\n',
-    '\t<entry rel-src="/baz.html">\n\t\t<title>baz</title>\n\t</entry>\n',
+    '\t<entry rel-src="/foo.html">\n\t\t<title><![CDATA[foo]]></title>\n\t</entry>\n',
+    '\t<entry rel-src="/bar.html">\n\t\t<title><![CDATA[bar]]></title>\n\t</entry>\n',
+    '\t<entry rel-src="/baz.html">\n\t\t<title><![CDATA[baz]]></title>\n\t</entry>\n',
     '</root>\n',
@@ -30,3 +30,3 @@ ].join('') });
   var title = root.find('title');
-  assert.equal(title , 'foo', 'returns correct tag content');
+  assert.equal(title , '<![CDATA[foo]]>', 'returns correct tag content');
   assert.equal(root.content() , title, '#content returns same content');
@@ -60,5 +60,5 @@ });
 test('fails when end of string is reached', function() {
-  assert.equal(root.find('title'), 'foo');
-  assert.equal(root.findNext('title'), 'bar');
-  assert.equal(root.findNext('title'), 'baz');
+  assert.equal(root.find('title'), '<![CDATA[foo]]>');
+  assert.equal(root.findNext('title'), '<![CDATA[bar]]>');
+  assert.equal(root.findNext('title'), '<![CDATA[baz]]>');
   assert(root.next(), 'moves to end of string');
@@ -76,3 +76,3 @@ var oldCursor = root.cursor;
   root.skip();
-  assert.equal(root.find('title'), 'bar', 'now starts with second entry');
+  assert.equal(root.find('title'), '<![CDATA[bar]]>', 'now starts with second entry');
 });
@@ -85,3 +85,3 @@
   root.skip();
-  assert.equal(root.find('title'), 'baz', 'now starts with third entry');
+  assert.equal(root.find('title'), '<![CDATA[baz]]>', 'now starts with third entry');
 });
@@ -94,4 +94,5 @@
   root.transformContent('title', { from: /f(oo)/, to: 'b$1' });
-  assert.equal(root.find('title'), 'boo', 'partially replaces content');
+  assert.equal(root.find('title'), '<![CDATA[boo]]>', 'partially replaces content');
 
+  root.transformContent('title', { to: 'boo' });
   root.transformContent('title', { to: '$& boo' });
@@ -102,7 +103,7 @@ assert.equal(root.find('title'), 'boo boo', "defaults 'from' to full content");
 test('transforms content of successive tags reliably', function() {
-  root.transformContent('title', { to: '$& (foo)' }).next();
-  root.transformContent('title', { to: '$& (bar)' }).next();
-  root.transformContent('title', { to: '$& (baz)' });
+  root.transformContent('title', { from: /foo/, to: '$& (foo)' }).next();
+  root.transformContent('title', { from: /bar/, to: '$& (bar)' }).next();
+  root.transformContent('title', { from: /baz/, to: '$& (baz)' });
   root.cursor = 0;
-  assert.equal(root.find('title'), 'foo (foo)', 'content remains as intended');
+  assert.equal(root.find('title'), '<![CDATA[foo (foo)]]>', 'content remains as intended');
 });
@@ -109,0 +110,0 @@

No alert changes

Improved metrics

Total package byte prevSize

49687

increased by16.68%

Number of package files

30

increased by7.14%

Lines of code

1356

increased by14.14%

No dependency changes