kissmetrics-export - npm Package Compare versions

Comparing version 0.1.7 to 0.2.0

lib/index.js

 var S3 = require('./s3');
-var Stream = require('./stream');
+var ObjectStream = require('./object_stream');
+var FilterStream = require('./filter_stream');
+var EventStream = require('./event_stream');
 var split = require('split');
@@ -22,6 +24,8 @@
 	var stream = this;
-	this._readStream = new Stream(this.s3, parameters);
-	return this._readStream.pipe(split(stream.parser));
+	this._objectStream = new ObjectStream(this.s3);
+	this._filterStream = new FilterStream(parameters);
+	this._eventStream = new EventStream(this.s3);
+	return this._objectStream.pipe(this._filterStream).pipe(this._eventStream).pipe(split(stream.parser));
 };
 
 module.exports = KmExport;

lib/s3.js

 var aws = require('aws-sdk');
 var Q = require('q');
-var _ = require('lodash');
-var moment = require('moment');
 
@@ -22,42 +20,15 @@ function KmS3(config) {
 /** todo: support fetching more than 1000 objects */
-KmS3.prototype.listS3Objects = function(dateRange) {
-	var kmp = this;
-	var deferred = Q.defer();
+KmS3.prototype.listS3Objects = function(marker) {
+	var self = this;
 	var params = {
 		Bucket: this.bucket,
-		Prefix: 'revisions/'
+		Prefix: 'revisions/',
+		Marker: marker
 	};
-	this.s3.listObjects(params, function(err, data) {
-		if (err) {
-			return deferred.reject(err); // an error occurred
-		}
-		else {
-			var objects = kmp.filterS3ObjectsByDate(data.Contents, dateRange);
-			return deferred.resolve(objects); // successful response
-		}
+	return Q.ninvoke(self.s3, 'listObjects', params).then(function(data) {
+		return data;
 	});
-	return deferred.promise;
 };
 
-KmS3.prototype.filterS3ObjectsByDate = function(contents, dateRange) {
-	if (typeof dateRange === 'undefined') {
-		return contents;
-	}
-	if (!_.isDate(dateRange.fromDate) || !_.isDate(dateRange.toDate)) {
-		throw Error('Not a valid date range');
-	}
-	var fromDate = moment(dateRange.fromDate).valueOf();
-	var toDate = moment(dateRange.toDate).valueOf();
-	return _.chain(contents).map(function(object) {
-		var lastModified = moment(object.LastModified).valueOf();
-		if (lastModified >= fromDate && lastModified <= toDate) {
-			return object.Key;
-		} else {
-			return;
-		}
-	}).compact().value();
-};
-
 KmS3.prototype.getS3Object = function(key) {
-	var deferred = Q.defer();
 	var params = {
@@ -67,21 +38,6 @@ Bucket: this.bucket,
 	};
-	this.s3.getObject(params, function(err, data) {
-		if (err) {
-			return deferred.reject(err); // an error occurred
-		}
-		else {
-			return deferred.resolve(data); // successful response
-		}
-	});
-	return deferred.promise;
+	var s3 = this.s3;
+	return Q.ninvoke(s3, 'getObject', params);
 };
 
-KmS3.prototype.createStreamfromObject = function(key) {
-	var params = {
-		Bucket: this.bucket,
-		Key: key
-	};
-	return this.s3.getObject(params).createReadStream();
-};
-
 module.exports = KmS3;

package.json

 {
   "name": "kissmetrics-export",
-  "version": "0.1.7",
+  "version": "0.2.0",
   "description": "Kissmetrics parser in node.js.",
@@ -24,2 +24,3 @@ "homepage": "https://github.com/MadKudu/kissmetrics-export",
     "aws-sdk": "^2.1.21",
+    "debug": "^2.1.3",
     "lodash": "^3.6.0",
@@ -33,2 +34,3 @@ "moment": "^2.9.0",
     "chai": "^2.2.0",
+    "from2": "^1.3.0",
     "grunt-cli": "^0.1.13",
@@ -35,0 +37,0 @@ "grunt-contrib-jshint": "^0.10.0",

test/test_s3.js

 /* globals describe, it, before*/
 (function () {
 	'use strict';
-	var _ = require('lodash');
 	var chai = require('chai');
 	// var chaiAsPromised = require('chai-as-promised');
 	// chai.use(chaiAsPromised);
-	var should = require('chai').should();
 	var expect = chai.expect;
@@ -17,21 +15,2 @@
 
-	var options = {};
-
-	var contents = [{
-			Key: 'revisions/1.json',
-			LastModified: new Date('2015-04-01'),
-			ETag: '1e1e90e47dddd56ab3b3771231a2aa49',
-			Size: 1199,
-			StorageClass: 'STANDARD',
-			Owner: {}
-		},
-		{
-			Key: 'revisions/10.json',
-			LastModified: new Date('2015-04-05'),
-			ETag: '"2949394731faf129789c6b61ff447b10"',
-			Size: 303,
-			StorageClass: 'STANDARD',
-			Owner: {}
-	}];
-
 	describe('KmS3', function () {
@@ -43,54 +22,10 @@ var KmS3 = require('../lib/s3.js');
 		});
-		describe('filterS3ObjectsByDate', function() {
-			it('should return the entire content if no range is provided', function () {
-				expect(kmS3.filterS3ObjectsByDate(contents)).to.have.length(2);
-			});
-			it('should throw an error if not a valid date range', function () {
-				var dateRange = {
-					fromDate: 'abc',
-					toDate: new Date('2015-04-03')
-				};
-				try {
-					kmS3.filterS3ObjectsByDate(contents, dateRange);
-				} catch (e) {
-					expect(_.isError(e)).to.be.true;
-				}
-			});
-			it('should return two objects', function () {
-				var dateRange = {
-					fromDate: new Date('2015-03-31'),
-					toDate: new Date('2015-04-30')
-				};
-				expect(kmS3.filterS3ObjectsByDate(contents, dateRange)).to.have.length(2);
-			});
-			it('should return no object', function () {
-				var dateRange = {
-					fromDate: new Date('2015-03-01'),
-					toDate: new Date('2015-03-31')
-				};
-				expect(kmS3.filterS3ObjectsByDate(contents, dateRange)).to.have.length(0);
-			});
-			it('should return one object', function () {
-				var dateRange = {
-					fromDate: new Date('2015-04-01'),
-					toDate: new Date('2015-04-03')
-				};
-				expect(kmS3.filterS3ObjectsByDate(contents, dateRange)).to.have.length(1);
-			});
-		});
 		describe('listS3Objects', function () {
-			it('should list objects in the bucket', function () {
-				return kmS3.listS3Objects().then(function(data) {
+			it('should list all objects in the bucket', function (done) {
+				this.timeout(10000);
+				kmS3.listS3Objects().then(function(data) {
 					expect(typeof data).to.not.equal('undefined');
+					done();
 				});
 			});
-			it('should list objects with a date range', function () {
-				var dateRange = {
-					fromDate: new Date('2015-03-08'),
-					toDate: new Date('2015-03-09')
-				};
-				return kmS3.listS3Objects(dateRange).then(function(data) {
-					expect(typeof data).to.not.equal('undefined');
-				});
-			});
 		});
@@ -97,0 +32,0 @@ describe('getS3Object', function () {

test/test.js

@@ -18,6 +18,2 @@ /* globals describe, it, before*/
 	describe('KmProcessor', function () {
-		var parameters = {
-			fromDate: new Date('2015-04-01'),
-			toDate: new Date('2015-04-03')
-		};
 		var KmExport = require('../lib/index');
@@ -40,8 +36,13 @@ var kmExport = new KmExport(config);
 		});
-		describe('parsed_stream', function() {
+		describe('stream', function() {
+			var parameters = {
+				fromDate: new Date('2015-04-07 04:00:00'),
+				toDate: new Date('2015-04-07 05:00:00')
+			};
 			var stream = kmExport.stream(parameters);
 			it('should stream data', function (done) {
 				this.timeout('60000');
+				var counter = 0;
 				stream.on('data', function(data) {
-					console.log(data);
+					counter += 1;
 					expect(data).to.be.not.null;
@@ -54,2 +55,3 @@ });
 				stream.on('end', function() {
+					//expect(counter).to.equal(47);
 					console.log('done');
@@ -56,0 +58,0 @@ done();

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

748649

increased by0.75%

Number of package files

21

increased by31.25%

Lines of code

6890

increased by2.64%

Worsened metrics

Dependency count

8

increased by14.29%

Dev dependency count

11

increased by10%

Number of low supply chain risk alerts

15

increased by150%

Dependency changes

• + Addeddebug@^2.1.3

• + Addeddebug@2.6.9(transitive)

• + Addedms@2.0.0(transitive)