Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

fiftyone.pipeline.core

Package Overview
Dependencies
Maintainers
1
Versions
124
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

fiftyone.pipeline.core - npm Package Compare versions

Comparing version 4.1.0 to 4.1.1

examples/customFlowElements/clientSideEvidenceFlowElement.js
examples/customFlowElements/simpleEvidenceFlowElement.js
tests/coreTestSetup.js
tests/evidence.test.js
tests/logs-errors-and-stops.test.js

22

.eslintrc.json

@@ -7,2 +7,5 @@ {

},
"plugins": [
"jsdoc"
],
"extends": [

@@ -30,4 +33,21 @@ "standard"

"always"
]
],
"jsdoc/check-alignment": 1,
"jsdoc/check-param-names": 1,
"jsdoc/check-tag-names": 1,
"jsdoc/check-types": 1,
"jsdoc/implements-on-classes": 1,
"jsdoc/newline-after-description": 1,
"jsdoc/require-description": 1,
"jsdoc/require-jsdoc": 1,
"jsdoc/require-param": 1,
"jsdoc/require-param-description": 1,
"jsdoc/require-param-name": 1,
"jsdoc/require-param-type": 1,
"jsdoc/require-returns": 1,
"jsdoc/require-returns-check": 1,
"jsdoc/require-returns-description": 1,
"jsdoc/require-returns-type": 1,
"jsdoc/valid-types": 1
}
}

31

aspectPropertyValue.js

@@ -23,6 +23,23 @@ /* *********************************************************************

/**
* An AspectPropertyValue is a wrapper for a value
* It lets you check this wrapper has a value inside it
* If not value is set, a specific no value message is returned
*/
class AspectPropertyValue {
constructor (noValueMessage = 'No value has been set.') {
this.noValueMessage = noValueMessage;
this.value = undefined;
/**
* Constructor for AspectPropertyValue
*
* @param {string} noValueMessage The message to show when no value is set
* @param {Mixed} value The value inside the wrapper
*/
constructor (noValueMessage = 'No value has been set.', value) {
if (typeof value !== 'undefined') {
this.noValueMessage = null;
this._value = value;
this.hasValue = true;
} else {
this.noValueMessage = noValueMessage;
this.hasValue = false;
}
}

@@ -32,3 +49,4 @@

* Get the value of this instance.
* @returns {Mixed}
*
* @returns {Mixed} The value of the property
* @throws Will throw error if 'hasValue' is false.

@@ -46,6 +64,7 @@ */

* Set the value of this instance.
* @param {Mixed} value
*
* @param {Mixed} value the value to set
*/
set value (value) {
if (typeof value !== 'undefined' && value !== null) {
if (typeof value !== 'undefined') {
this._value = value;

@@ -52,0 +71,0 @@ this.hasValue = true;

5

basicListEvidenceKeyFilter.js

@@ -32,2 +32,3 @@ /* *********************************************************************

* constructor for basicListEvidenceKeyFilter
*
* @param {Array} list items to keep when filtering

@@ -42,4 +43,6 @@ */

/**
* Check if a specific evidence key should be filtered
*
* @param {string} key to check in the filter
* @return {boolean} is this key in the filter's keys list?
* @returns {boolean} is this key in the filter's keys list?
*/

@@ -46,0 +49,0 @@ filterEvidenceKey (key) {

41

elementData.js

@@ -23,7 +23,13 @@ /* *********************************************************************

/**
* Stores information created by a flowElement based on flowData.
* Stored in flowData
*/
class ElementData {
/**
* constructor for elementData, stores information created by a flowElement based on flowData. Stored (keyed by flowElement and searchable via meta data properties) in flowData
* @param {Object} options
* @param {flowElement} options.flowElement
* Constructor for elementData
*
* @param {object} options the options object
* @param {flowElement} options.flowElement the FlowElement this data
* is part of
*/

@@ -37,2 +43,4 @@ constructor ({ flowElement }) {

// Returns a proxy so that we can use flowData.flowElementDataKey
// As a shortcut to flowData.get("flowElementDataKey")
return new Proxy(this, {

@@ -52,3 +60,5 @@ get: (data, key) => {

* called via the elementData's get method
* @param {String} key
* This method is overriden by classes inheriting from elementData
*
* @param {string} key the key to retrieve a property value for
*/

@@ -60,5 +70,7 @@ getInternal (key) {

/**
* A wrapper that performs actions before passing on processing (or skipping) the getInternal method
* @param {String} key
* @return {mixed} value
* A wrapper that performs actions before passing on processing
* (or skipping) the getInternal method
*
* @param {string} key the key to retreive a property value for
* @returns {mixed} value
*/

@@ -71,4 +83,5 @@ get (key) {

* Return string value of property
* @param {String} key
* @return {string} value
*
* @param {string} key the key to retreive a property value for
* @returns {string} value
*/

@@ -85,4 +98,5 @@ getAsString (key) {

* Return float value of property
* @param {String} key
* @return {float} value
*
* @param {string} key the key to retreive a property value for
* @returns {float} value
*/

@@ -99,4 +113,5 @@ getAsFloat (key) {

* Return int value of property
* @param {String} key
* @return {int} value
*
* @param {string} key the key to retreive a property value for
* @returns {int} value
*/

@@ -103,0 +118,0 @@ getAsInteger (key) {

22

elementDataDictionary.js

@@ -25,7 +25,12 @@ /* *********************************************************************

/**
* A basic extension of elementData with dictionary object storage / lookup
**/
class ElementDataDictionary extends ElementData {
/**
* a basic extension of elementData with dictionary object storage / lookup
* @param {Object} options
* @param {flowElement} options.flowElement
* Constructor for ElementDataDictionary
*
* @param {object} options options object
* @param {FlowElement} options.flowElement FlowElement the options are
* added to
* @param {contents} options.contents // key value dictionary

@@ -39,6 +44,13 @@ */

* [Symbol.iterator] () {
for (let i = 0; i < Object.keys(this.contents).length; i += 1) {
yield Object.keys(this.contents)[i];
}
}
/**
* Return value from elementData dictionary
* @param {String} key
* @return {int} value
*
* @param {string} key the property key to retrieve
* @returns {int} value
*/

@@ -45,0 +57,0 @@ getInternal (key) {

81

evidence.js

@@ -25,41 +25,10 @@ /* *********************************************************************

(function () {
const reduce = Function.bind.call(Function.call, Array.prototype.reduce);
const isEnumerable = Function.bind.call(
Function.call,
Object.prototype.propertyIsEnumerable
);
const concat = Function.bind.call(Function.call, Array.prototype.concat);
const keys = Reflect.ownKeys;
if (!Object.values) {
Object.values = function values (O) {
return reduce(
keys(O),
(v, k) =>
concat(v, typeof k === 'string' && isEnumerable(O, k) ? [O[k]] : []),
[]
);
};
}
if (!Object.entries) {
Object.entries = function entries (O) {
return reduce(
keys(O),
(e, k) =>
concat(
e,
typeof k === 'string' && isEnumerable(O, k) ? [[k, O[k]]] : []
),
[]
);
};
}
})();
/**
* Storage of evidence on a flowData object
*/
class Evidence {
/**
* Storage of evidence on a flowData object
* @param {FlowData} flowData
* Constructor for evidence
*
* @param {FlowData} flowData FlowData to add the evidence to
*/

@@ -73,4 +42,5 @@ constructor (flowData) {

* Add a piece of evidence to flowData
* @param {String} key
* @param {Mixed} value
*
* @param {string} key evidence key to add
* @param {Mixed} value value of evidence key
*/

@@ -104,5 +74,6 @@ add (key, value) {

* Add a piece of evidence to flowData as an object
* @param {Object} evidenceObject
* @param {String} evidenceObject.key
* @param {Mixed} evidenceObject.value
*
* @param {object} evidenceObject key value map of evidence
* @param {string} evidenceObject.key evidencekey
* @param {Mixed} evidenceObject.value evidence value
*/

@@ -112,4 +83,4 @@ addObject (evidenceObject) {

Object.entries(evidenceObject).forEach(function ([key, value]) {
evidenceContainer.add(key, value);
Object.keys(evidenceObject).forEach(function (key) {
evidenceContainer.add(key, evidenceObject[key]);
});

@@ -119,4 +90,8 @@ }

/**
* Add a piece of evidence to flowData from an HTTP request
* @param {Object} request
* Add evidence to flowData from an HTTP request
* This helper automatically adds evidence:
* headers, cookies, protocol, IP and query params
*
* @param {object} request an HTTP request object
* @returns {undefined}
*/

@@ -128,3 +103,4 @@ addFromRequest (request) {

Object.entries(request.headers).forEach(([key, value]) => {
Object.keys(request.headers).forEach(key => {
const value = request.headers[key];
let requestHeaderKey;

@@ -183,3 +159,4 @@ let requestHeaderValue;

Object.entries(query).forEach(function ([key, value]) {
Object.keys(query).forEach(function (key) {
const value = query[key];
evidence.add('query.' + key, value);

@@ -193,4 +170,5 @@ });

* Get a piece of evidence
* @param {String} key
* @returns {Mixed}
*
* @param {string} key evidence key to retreive
* @returns {mixed} the evidence value
*/

@@ -203,3 +181,4 @@ get (key) {

* Get all evidence
* @returns {Object} all evidence
*
* @returns {object} all evidence
*/

@@ -206,0 +185,0 @@ getAll () {

19

evidenceKeyFilter.js

@@ -23,6 +23,15 @@ /* *********************************************************************

/**
* An evidence key filter is added to a flowElement
* It tells the pipeline which evidence it is interested in
* This can be used to determine whether a request can be cached
* Or to filter out evidence not needed by any element in a pipeline
* This base class is always extended for a specific filter type
*/
class EvidenceKeyFilter {
/**
* Filter an evidence object
* @param {Object} evidenceKeyObject
*
* @param {object} evidenceKeyObject key value object of evidence
* @returns {object} filtered evidence object
*/

@@ -44,5 +53,7 @@ filterEvidence (evidenceKeyObject) {

/**
* Internal filterEvidenceKey function overriden by specific filters to keep or filter out a piece of evidence
* @param {String} evidenceKey
* @returns {Boolean}
* Internal filterEvidenceKey function overriden
* by specific filters to keep or filter out a piece of evidence
*
* @param {string} key the evidence key to check
* @returns {boolean} whether the key should stay or not
*/

@@ -49,0 +60,0 @@ filterEvidenceKey (key) {

11

examples/examples.test.js

@@ -59,7 +59,14 @@ /* *********************************************************************

test('custom flowElements 1', () => {
testExample({ file: __dirname + '/customFlowElements/1-simpleEvidenceFlowElement.js' });
testExample(
{ file: __dirname + '/customFlowElements/simpleEvidenceFlowElement.js' }
);
});
test('custom flowElements 2', () => {
testExample({ file: __dirname + '/customFlowElements/2-clientSideEvidenceFlowElement.js', port: 3001 });
testExample(
{
file: __dirname + '/customFlowElements/clientSideEvidenceFlowElement.js',
port: 3001
}
);
});

4

examples/flowElementsForExamples/async.js

@@ -54,3 +54,5 @@ /* *********************************************************************

const data = new ElementDataDictionary({ flowElement: flowElement, contents: contents });
const data = new ElementDataDictionary(
{ flowElement: flowElement, contents: contents }
);

@@ -57,0 +59,0 @@ flowData.setElementData(data);

7

examples/flowElementsForExamples/neverRun.js

@@ -23,3 +23,4 @@ /* *********************************************************************

// This flowElement is a dummy for putting in a pipeline after the stopElement (demonstrating that it will never process)
// This flowElement is a dummy for putting in a pipeline
// after the stopElement (demonstrating that it will never process)

@@ -43,3 +44,5 @@ // Note that this example is designed to be run from within the

processInternal (flowData) {
const data = new ElementDataDictionary({ flowElement: this, contents: { no: false } });
const data = new ElementDataDictionary(
{ flowElement: this, contents: { no: false } }
);

@@ -46,0 +49,0 @@ flowData.setElementData(data);

12

examples/flowElementsForExamples/sync.js

@@ -23,3 +23,5 @@ /* *********************************************************************

// This flowElement runs syncronously and checks the value of the async flowElement in order to demonstrate sync vs async pipelines
// This flowElement runs syncronously and checks the
// value of the async flowElement in order to demonstrate
// sync vs async pipelines

@@ -41,3 +43,5 @@ // Note that this example is designed to be run from within the

this.dataKey = 'sync';
this.evidenceKeyFilter = new BasicListEvidenceKeyFilter(['header.user_agent']);
this.evidenceKeyFilter = new BasicListEvidenceKeyFilter(
['header.user_agent']
);

@@ -63,3 +67,5 @@ this.properties = {

const data = new ElementDataDictionary({ flowElement: this, contents: contents });
const data = new ElementDataDictionary(
{ flowElement: this, contents: contents }
);

@@ -66,0 +72,0 @@ flowData.setElementData(data);

24

examples/pipelines.js

@@ -26,5 +26,6 @@ /* *********************************************************************

This example demonstrates syncronous and asyncronous pipelines with multiple flowElements and other core features of the 51Degrees pipeline
This example demonstrates syncronous and asyncronous pipelines
with multiple flowElements and other core features of the 51Degrees pipeline
*/
*/

@@ -46,3 +47,4 @@ // Note that this example is designed to be run from within the

// A flowElement that runs syncronously which loads a different result if the async one has already finished processing
// A flowElement that runs syncronously which loads a different
// result if the async one has already finished processing

@@ -55,3 +57,4 @@ const sync = new (require('./flowElementsForExamples/sync'))();

// A dummy flowElement to show that subsequent flowElements don't process once the flowData has been stopped
// A dummy flowElement to show that subsequent flowElements
// don't process once the flowData has been stopped
const neverRun = new (require('./flowElementsForExamples/neverRun'))();

@@ -61,3 +64,8 @@

const syncPipeline = new PipelineBuilder().add(async).add(sync).add(stop).add(neverRun).build();
const syncPipeline = new PipelineBuilder()
.add(async)
.add(sync)
.add(stop)
.add(neverRun)
.build();

@@ -87,6 +95,8 @@ // Create flowData

// This is a pipeline that runs the flowElements in parallel (the sync element will not wait for the async one to finish)
// This is a pipeline that runs the flowElements in parallel
// (the sync element will not wait for the async one to finish)
const asyncPipeline = new PipelineBuilder().addParallel([async, sync]);
const asyncPipeline = new PipelineBuilder()
.addParallel([async, sync]);
asyncPipeline.build();

110

flowData.js

@@ -25,37 +25,8 @@ /* *********************************************************************

(function () {
const reduce = Function.bind.call(Function.call, Array.prototype.reduce);
const isEnumerable = Function.bind.call(
Function.call,
Object.prototype.propertyIsEnumerable
);
const concat = Function.bind.call(Function.call, Array.prototype.concat);
const keys = Reflect.ownKeys;
if (!Object.values) {
Object.values = function values (O) {
return reduce(
keys(O),
(v, k) =>
concat(v, typeof k === 'string' && isEnumerable(O, k) ? [O[k]] : []),
[]
);
};
}
if (!Object.entries) {
Object.entries = function entries (O) {
return reduce(
keys(O),
(e, k) =>
concat(
e,
typeof k === 'string' && isEnumerable(O, k) ? [[k, O[k]]] : []
),
[]
);
};
}
})();
/**
* FlowData is created by a specific pipeline
* It collects evidence set by the user
* It passes evidence to flowElements in the pipeline
* These elements can return ElementData or populate an errors object
*/
class FlowData {

@@ -65,3 +36,4 @@ /**

* Called by a pipeline's createFlowData() method
* @param {Pipeline} pipeline
*
* @param {Pipeline} pipeline the pipeline to add the FlowData to
*/

@@ -71,3 +43,4 @@ constructor (pipeline) {

// All evidence added to a flowData is stored in a special collection class for easy retrieval and searching
// All evidence added to a flowData is stored in a special
// collection class for easy retrieval and searching
this.evidence = new Evidence(this);

@@ -78,3 +51,4 @@

// Stop flag, if the flowData is stopped, no further flowElements are processed
// Stop flag, if the flowData is stopped, no further
// flowElements are processed
this.stopped = false;

@@ -87,3 +61,6 @@

/**
* Get back an object that lists all evidence that will be used by any flowElements in the flowData's parent pipeline
* Get back an object that lists all evidence that will be
* used by any flowElements in the flowData's parent pipeline
*
* @returns {object} Evidence that is requested by flowElements
*/

@@ -94,6 +71,7 @@ getEvidenceDataKey () {

// Keep list of evidence that is requested by flowElements
// Evidence that is requested by flowElements
let keep = {};
Object.values(this.pipeline.flowElements).forEach(function (flowElement) {
Object.keys(this.pipeline.flowElements).forEach(function (key) {
const flowElement = this.pipeline.flowElements[key];
// merge in the filtered evidence for the flowElement

@@ -119,3 +97,7 @@ keep = Object.assign(

/**
* Set an error on the flowData (usually triggered by something going wrong in a flowElement's process function)
* Set an error on the flowData (usually triggered by
* something going wrong in a flowElement's process function)
*
* @param {mixed} error the error to throw
* @param {FlowElement} flowElement the FlowElement the error is thrown on
*/

@@ -143,3 +125,4 @@ setError (error, flowElement) {

* Processes the flowData (running the process methods on all connected)
* @returns {Promise}
*
* @returns {Promise} result of processing
*/

@@ -153,5 +136,7 @@ process () {

/**
* Add to the flowData object with a class derived from elementData
* @param {elementData}
* @returns {FlowData}
* Add to the flowData object with a class derived from ElementData
*
* @param {ElementData} data instance of ElementData to
* set for the FlowElement's datakey
* @returns {FlowData} the FlowData object
*/

@@ -165,4 +150,6 @@ setElementData (data) {

/**
* Get elementData by flowElementDataKey
* @param {string} flowElementDataKey
* Get ElementData by a flowElement's data key
*
* @param {string} flowElementDataKey the datakey of a FlowElement
* @returns {ElementData} data from the FlowElement
*/

@@ -176,4 +163,6 @@ get (flowElementDataKey) {

/**
* get elementData by a flowElement object
* @param {FlowElement}
* get ElementData by a FlowElement object
*
* @param {FlowElement} flowElement The FlowElement to fetch data for
* @returns {ElementData} data from the FlowElement
*/

@@ -187,5 +176,10 @@ getFromElement (flowElement) {

/**
* get an object ({key:value} store) of elementData based on a metadata key and value, alternatively pass in a filtering function to manually filter available propeties
* @param {String} metaKey
* @param {String|Function} metaValueorFuncton value or a filter function which receives the value of the metaKey and returns a boolean
* get an object ({key:value} store) of elementData
* based on a metadata key and value, alternatively
* pass in a filtering function to manually filter available propeties
*
* @param {string} metaKey a metakey such as "category"
* @param {string|Function} metaValueorFuncton value or a filter
* function which receives the value of the metaKey and returns a boolean
* @returns {object} key value pair of matching properties and values
*/

@@ -217,6 +211,4 @@ getWhere (metaKey, metaValueorFuncton) {

Object.entries(filteredProperties).forEach(function ([
propertyName,
property
]) {
Object.keys(filteredProperties).forEach(function (propertyName) {
const property = filteredProperties[propertyName];
const flowElementKey = property.flowElementKey;

@@ -227,3 +219,5 @@

} catch (e) {
// Ignore errors when fetching a property as getWhere would often crash to an exception if properties are not available, they should just be excluded
// Ignore errors when fetching a property
// as getWhere would often crash to an exception
// if properties are not available, they should be excluded
}

@@ -230,0 +224,0 @@ });

61

flowElement.js

@@ -25,2 +25,9 @@ /* *********************************************************************

/**
* A FlowElement is placed inside a pipeline
* It receives Evidence via a FlowData object
* It uses this to optionally create ElementData on the Flowdata
* It has a unique dataKey which is used to extract data from the FlowData
* Any errors in processing are caught in the FlowData's errors object
**/
class FlowElement {

@@ -30,7 +37,9 @@ /**

*
* @param {Object} options
* @param {String} options.dataKey the dataKey the flowElement's elementData will be stored under
* @param {object} options options for the FlowElement
* @param {string} options.dataKey the dataKey the flowElement's
* elementData will be stored under
* @param {Function} options.processInternal callback to act on flowData
* @param {Object} options.properties list of properties including metadata
* @param {{Function(new:EvidenceKeyFilter)}} options.evidenceKeyFilter an instance of an evidenceKeyFilter
* @param {object} options.properties list of properties including metadata
* @param {{EvidenceKeyFilter}} options.evidenceKeyFilter an instance of
* an EvidenceKeyFilter to filter evidence added to the Pipeline
*/

@@ -64,4 +73,9 @@ constructor ({

/**
* Function to be called when a flowElement is added to pipeline, runs through any registrationCallbacks on the flowElement
* */
* Internal function to be called when a FlowElement is added
* to pipeline, runs through any registrationCallbacks on the FlowElement
*
* @param {Pipeline} pipeline the Pipeline the FlowElement is registered with
* @param {FlowElement} flowElement The FlowElement the
* registration callback is called on
*/
onRegistration (pipeline, flowElement) {

@@ -74,5 +88,19 @@ this.registrationCallbacks.forEach(function (registrationCallback) {

/**
* Internal process function for a particular flowElement called (via the flowElement.process() method) when flowData generated by a pipleline is processsed. Overriden by instances of this base class
* Function called to check if a FlowElement is ready
* Used when there are asynchronous initialisation steps
* @returns {Promise}
* */
ready () {
return Promise.resolve(this);
}
/**
* Internal process function for a particular flowElement called
* (via the flowElement.process() method) when flowData generated
* by a pipleline is processsed. Overriden by instances of this base class
*
* @param {FlowData} flowData FlowData being processed
* @returns {Mixed} result of processing
*/
processInternal () {
processInternal (flowData) {
return true;

@@ -82,3 +110,8 @@ }

/**
* To allow actions to take place before and after a flowElement's processInternal function runs, a process wrapper is run first
* To allow actions to take place before and after a
* FlowElement's processInternal function runs, a process
* wrapper is run first
*
* @param {FlowData} flowData FlowData being processed
* @returns {Promise} FlowData after processing
*/

@@ -90,3 +123,6 @@ process (flowData) {

/**
* Call this function to update the properties meta database in all the pipelines this flowElement has been added to
* Call this function to update the properties meta database
* in all the pipelines this flowElement has been added to
*
* @returns {Promise} notification of complete updates
*/

@@ -106,4 +142,7 @@ updateProperties () {

/**
* Get a flowElement's properties. By default returns a promise wrapped version of the object's properties list
* Get a flowElement's properties. By default returns a
* promise wrapped version of the object's properties list
* Can return standard value or promise
*
* @returns {object} dictionary of properties
*/

@@ -110,0 +149,0 @@ getProperties () {

72

javascriptbuilder.js

@@ -37,2 +37,7 @@ /* *********************************************************************

/**
* An instance of EvidenceKeyFilter which removes all but header
* and query evidence as that is all that is used by
* the JavaScript builder
**/
class JSEvidenceKeyFilter extends EvidenceKeyFilter {

@@ -44,10 +49,39 @@ filterEvidenceKey (key) {

/**
* The JavaScriptBuilder aggregates JavaScript properties
* from FlowElements in the pipeline. This JavaScript also
* (when needed) generates a fetch request to retrieve additional properties
* populated with data from the client side
* It depends on the JSON Bundler element
* (both are automatically added to a pipeline unless
* specifically removed) for its list of properties.
* The results of the JSON Bundler should also be used in a
* user-specified endpoint which retrieves the JSON from the
* client side. The JavaScriptBuilder is constructed with a
* url for this endpoint.
*/
class JavaScriptBuilderElement extends FlowElement {
/**
* Constructor for JavaScriptBuilder.
*
* @param {object} options options object
* @param {string} options.objName the name of the client
* side object with the JavaScript properties in it
* @param {string} options.protocol The protocol ("http" or "https")
* used by the client side callback url.
* This can be overriden with header.protocol evidence
* @param {string} options.host The host of the client side
* callback url. This can be overriden with header.host evidence.
* @param {string} options.endPoint The endpoint of the client side
* callback url
* @param {boolean} options.enableCookies whether cookies should be enabled
* @param {boolean} options.minify Whether to minify the JavaScript
*/
constructor ({
_objName = 'fod',
_protocol = '',
_host = '',
_endPoint = '',
_enableCookies = true,
_minify = true
objName = 'fod',
protocol = '',
host = '',
endPoint = '',
enableCookies = true,
minify = true
} = {}) {

@@ -57,8 +91,8 @@ super(...arguments);

this.settings = {
_objName: _objName,
_protocol: _protocol,
_host: _host,
_endPoint: _endPoint,
_enableCookies: _enableCookies,
_minify: _minify
objName: objName,
protocol: protocol,
host: host,
endPoint: endPoint,
enableCookies: enableCookies,
minify: minify
};

@@ -71,4 +105,8 @@

/**
* The JavaScriptBuilder serves JavaScript properties and allows for a fetch request to retrieve additional properties populated with data from the client side
* @param {flowData} flowData
* Internal process function of the JavaScript builder
* Gets JSON from the JSONBundler and constructs JavaScript
* to place on the client side
*
* @param {FlowData} flowData to process
* @returns {undefined}
*/

@@ -83,8 +121,8 @@ processInternal (flowData) {

for (const setting in this.settings) {
settings[setting] = this.settings[setting];
settings["_" + setting] = this.settings[setting];
}
// Generate url from parts
let protocol = this.settings._protocol;
let host = this.settings._host;
let protocol = this.settings.protocol;
let host = this.settings.host;

@@ -91,0 +129,0 @@ if (!protocol) {

20

jsonbundler.js

@@ -27,2 +27,11 @@ /* *********************************************************************

/**
* The JSONBundler aggregates all properties from FlowElements
* into a JSON object
* It is used for retrieving via an endpoint from the client
* side via the JavaScriptBuilder and also used inside the
* JavaScriptBuilder itself to pass properties to the client side.
* Both this and the JavaScriptBuilder element are automatically
* added to a pipeline unless specifically ommited in the PipelineBuilder
*/
class JSONBundlerElement extends FlowElement {

@@ -38,3 +47,4 @@ constructor () {

* The JSON Builder extracts all properties and serializes them into JSON
* @param {FlowData} flowData
*
* @param {FlowData} flowData the FlowData being processed
*/

@@ -59,3 +69,3 @@ processInternal (flowData) {

// Create empty area for flowElement properties to go
output[flowElement] = {};
output[flowElement.toLowerCase()] = {};

@@ -97,5 +107,5 @@ const flowElementObject = flowData.pipeline.flowElements[flowElement];

output[flowElement][property] = value;
output[flowElement.toLowerCase()][property.toLowerCase()] = value;
if (value == null) {
output[flowElement][property + 'nullreason'] = nullReason;
output[flowElement.toLowerCase()][property.toLowerCase() + 'nullreason'] = nullReason;
}

@@ -118,3 +128,3 @@

if (value) {
output.javascriptProperties.push(flowElement + '.' + property);
output.javascriptProperties.push(flowElement.toLowerCase() + '.' + property.toLowerCase());
}

@@ -121,0 +131,0 @@ }

3

package.json
{
"name": "fiftyone.pipeline.core",
"version": "4.1.0",
"version": "4.1.1",
"description": "Core library for the 51Degrees Pipeline API",

@@ -21,2 +21,3 @@ "directories": {

"eslint-plugin-import": "^2.20.2",
"eslint-plugin-jsdoc": "^25.4.0",
"eslint-plugin-node": "^11.1.0",

@@ -23,0 +24,0 @@ "eslint-plugin-promise": "^4.2.1",

118

pipeline.js

@@ -26,42 +26,13 @@ /* *********************************************************************

(function () {
const reduce = Function.bind.call(Function.call, Array.prototype.reduce);
const isEnumerable = Function.bind.call(
Function.call,
Object.prototype.propertyIsEnumerable
);
const concat = Function.bind.call(Function.call, Array.prototype.concat);
const keys = Reflect.ownKeys;
if (!Object.values) {
Object.values = function values (O) {
return reduce(
keys(O),
(v, k) =>
concat(v, typeof k === 'string' && isEnumerable(O, k) ? [O[k]] : []),
[]
);
};
}
if (!Object.entries) {
Object.entries = function entries (O) {
return reduce(
keys(O),
(e, k) =>
concat(
e,
typeof k === 'string' && isEnumerable(O, k) ? [[k, O[k]]] : []
),
[]
);
};
}
})();
/**
* Pipeline holding a list of flowElements for processing, can create
* flowData that will be passed through these, collecting elementData
* Should be constructed through the PipelineBuilder class
*/
class Pipeline {
/**
* Pipeline holding a list of flowElements for processing, can create flowData that will be passed through these, collecting elementData
* Should be constructed through the pipelineBuilder class
* @param {FlowElements[]}
* Constructor for Pipeline
*
* @param {FlowElements[]} flowElements list of FlowElements to
* add to the Pipeline
*/

@@ -80,3 +51,6 @@ constructor (flowElements = []) {

// Run through flowElements and store them by dataKey in the pipeline.flowElements object. Recursive function so it can handle parallel elements which are passed in as arrays
// Run through flowElements and store them by dataKey
// in the pipeline.flowElements object.
// Recursive function so it can handle parallel elements
// which are passed in as arrays
const storeInFlowElementList = function (flowElemmentList) {

@@ -100,11 +74,19 @@ flowElemmentList.forEach(function (slot) {

// Empty property database, later populated and possibly updated by flowElements' property lists
// Empty property database, later populated and possibly
// updated by flowElements' property lists
this.propertyDatabase = {};
// Update property list - Note that some of these could be async so the property list might not be updated straight away. When this happens, getWhere calls will simply return empty for those specific properties
Object.values(this.flowElements).forEach(function (flowElement) {
// Update property list - Note that some of these could be async
// so the property list might not be updated straight away.
// When this happens, getWhere calls will return
// empty for those specific properties
Object.keys(pipeline.flowElements).forEach(function (key) {
const flowElement = pipeline.flowElements[key];
pipeline.updatePropertyDataBaseForElement(flowElement);
});
// Create the flowData process function here to save having to do it each time the flowData is processed. This requestedPackages up all the logic for generating promises, sync / async and others
// Create the FlowData process function here to save
// having to do it each time the FlowData is processed.
// This packages up all the logic for generating promises,
// sync / async and others
const processMethod = function () {

@@ -181,4 +163,6 @@ const promiseChain = [];

/**
* get a flowElement by its dataKey
* @param {String} dataKey
* get a FlowElement by its dataKey
*
* @param {string} key the datakey of the FlowElement
* @returns {FlowElement} the FlowElement for the datakey
*/

@@ -192,4 +176,5 @@ getElement (key) {

* Shorthand access to the enclosed event emitter
* @param {String} listener type of message to listen to
* @param {Function} callback
*
* @param {string} listener type of message to listen to
* @param {Function} callback a callback to react to the log
*/

@@ -202,4 +187,5 @@ on (listener, callback) {

* Shorthand to trigger a message on the pipeline's eventEmitter
* @param {String} type type of message to listen to
* @param message
*
* @param {string} type type of message
* @param {mixed} message message to store in the log
*/

@@ -211,4 +197,7 @@ log (type, message) {

/**
* create a flowData element from the pipeline
* @returns {FlowData}
* Create a FlowData element from the pipeline
*
* @returns {FlowData} a FlowData object for the Pipeline
* containing methods for adding evidence and processing via
* the FlowElements in the Pipleine.
*/

@@ -218,3 +207,5 @@ createFlowData () {

// Create getters for flowElements by key so users can use flowData.elementDataKey instead of flowData.get(elementDataKey)
// Create getters for flowElements by key so users
// can use flowData.elementDataKey instead of
// flowData.get(elementDataKey)

@@ -239,7 +230,8 @@ Object.keys(this.flowElements).forEach(function (element) {

Object.keys(pipeline.propertyDatabase).forEach(function (metaKey) {
Object.entries(pipeline.propertyDatabase[metaKey]).forEach(function ([
metaValue,
list
]) {
Object.entries(list).forEach(function ([key, value]) {
Object.keys(pipeline.propertyDatabase[metaKey]).forEach(function (
metaValue
) {
const list = pipeline.propertyDatabase[metaKey][metaValue];
Object.keys(list).forEach(function (key) {
const value = list[key];
if (value.flowElementKey === flowElement.dataKey) {

@@ -255,11 +247,11 @@ delete pipeline.propertyDatabase[metaKey][metaValue][key];

Object.entries(properties).forEach(function ([
propertyKey,
propertyMeta
]) {
Object.keys(properties).forEach(function (
propertyKey
) {
const propertyMeta = properties[propertyKey];
const propertyName = propertyKey;
Object.entries(propertyMeta).forEach(function ([metaKey, metaValue]) {
Object.keys(propertyMeta).forEach(function (metaKey) {
metaKey = metaKey.toLowerCase();
metaValue = metaValue.toString().toLowerCase();
const metaValue = propertyMeta[metaKey].toString().toLowerCase();

@@ -266,0 +258,0 @@ if (!pipeline.propertyDatabase[metaKey]) {

51

pipelineBuilder.js

@@ -28,3 +28,22 @@ /* *********************************************************************

/**
* A PipelineBuilder generates a Pipeline object
* Before construction of the Pipeline, FlowElements are added to it
* There are also options for how JavaScript is output from the Pipeline
*/
class PipelineBuilder {
/**
* Constructor for pipeline builder
*
* @param {object} settings settings for the pipeline being
* constructed
* @param {boolean} settings.addJavaScriptBuilder Whether to
* automatically add the JSONBundler, JavaScriptBuilder
* and Sequence elements needed to output JavaScript
* from the system and generate an endpoint for fetching the properties
* generated by a pipeline from the client side. This is
* true by default.
* @param {object} settings.javascriptBuilderSettings The settings
* to pass to the JavaScriptBuilder. See JavaScriptBuilder class for details.
*/
constructor (settings = {}) {

@@ -45,4 +64,7 @@ this.flowElements = [];

/**
* Helper that loads a JSON configuration file from the filesystem and calls pipelineBuilder.buildFromConfiguration
* @param {String} path path to a JSON configuration file
* Helper that loads a JSON configuration file from
* the filesystem and calls pipelineBuilder.buildFromConfiguration
*
* @param {string} configPath path to a JSON configuration file
* @returns {Pipeline} the constructed pipeline
*/

@@ -59,3 +81,5 @@ buildFromConfigurationFile (configPath) {

* Create a pipeline from a JSON configuration
* @param {Object} config a JSON configuration object
*
* @param {object} config a JSON configuration object
* @returns {Pipeline} the constructed pipeline
*/

@@ -92,2 +116,10 @@ buildFromConfiguration (config) {

/**
* Internal function used to first check if the
* JavaScript elements should be added to the pipeline
* and add them if requested.
*
* @returns {FlowElement[]} list of JavaScript related
* FlowElements
*/
getJavaScriptElements () {

@@ -120,3 +152,6 @@ const flowElements = [];

* Add a single flowElement to be executed in series
* @param {FlowElement} flowElement
*
* @param {FlowElement} flowElement flowElement to add to the
* Pipeline
* @returns {PipelineBuilder} Pipeline builder for easy chaining
*/

@@ -131,3 +166,6 @@ add (flowElement) {

* Add an array of flowElements to be executed in parallel
* @param {FlowElement[]} flowElements
*
* @param {FlowElement[]} flowElements array of FlowElements
* to add to the Pipeline (to be exeuted in parallel)
* @returns {PipelineBuilder} Pipeline builder for easy chaining
*/

@@ -142,3 +180,4 @@ addParallel (flowElements) {

* Build the pipeline from the flowElements that have been added
* @returns {Pipeline}
*
* @returns {Pipeline} The constructed Pipeline
*/

@@ -145,0 +184,0 @@ build () {

15

sequenceElement.js

@@ -28,2 +28,9 @@ /* *********************************************************************

/**
* The SequenceElement stores session data regarding requests
* for client side JavaScript from the JavaScript created by a
* Pipeline's JavaScriptBuilder
* If a Pipeline is constructed with the JavaScript elements enabled
* this is added automatically along with the JavaScriptBuilder and JSONBundler.
**/
class SequenceElement extends FlowElement {

@@ -39,4 +46,8 @@ constructor () {

/**
* The SequenceElement stores session data for requests for JavaScript
* @param {FlowData} flowData
* Internal process function for the sequence element
* Checks if there is a session id and sequence number set
* if there is it increments the sequence number for that session
* if not it initialises both
*
* @param {FlowData} flowData flowData with the evidence in it
*/

@@ -43,0 +54,0 @@ processInternal (flowData) {

4

tests/configTestFlowElement.js

@@ -35,3 +35,5 @@ /* *********************************************************************

processInternal (flowData) {
const data = new ElementDataDictionary({ flowElement: this, contents: { built: this.prefix + '_world' } });
const data = new ElementDataDictionary(
{ flowElement: this, contents: { built: this.prefix + '_world' } }
);

@@ -38,0 +40,0 @@ flowData.setElementData(data);

178

tests/core.test.js

@@ -20,106 +20,16 @@ /* *********************************************************************

* in the end user terms of the application under an appropriate heading,
* such notice(s) shall fulfill the requirements of that article.
such notice(s) shall fulfill the requirements of that article.
* ********************************************************************* */
const FlowElement = require('../flowElement');
const setup = require(__dirname + '/coreTestSetup.js');
const PipelineBuilder = require('../pipelineBuilder');
const ElementDataDictionary = require('../elementDataDictionary');
const BasicListEvidenceKeyFilter = require('../basicListEvidenceKeyFilter');
const sync = new FlowElement({
dataKey: 'sync',
evidenceKeyFilter: new BasicListEvidenceKeyFilter(['header.user_agent']),
processInternal: function (flowData) {
const contents = { integer: 5 };
const syncPipeline = new PipelineBuilder()
.add(setup.async)
.add(setup.sync)
.build();
try {
contents.boolean = flowData.get('async').get('string') === 'hello';
} catch (e) {
contents.boolean = false;
}
const data = new ElementDataDictionary({ flowElement: this, contents: contents });
flowData.setElementData(data);
},
properties: {
integer: {
type: 'int'
},
boolean: {
type: 'bool'
}
}
});
const async = new FlowElement({
dataKey: 'async',
processInternal: function (flowData) {
const flowElement = this;
return new Promise(function (resolve, reject) {
setTimeout(function () {
const contents = { string: 'hello' };
const data = new ElementDataDictionary({ flowElement: flowElement, contents: contents });
flowData.setElementData(data);
resolve();
}, 500);
});
},
properties: {
string: {
type: 'string'
}
}
});
const error = new FlowElement({
dataKey: 'error',
processInternal: function (flowData) {
throw 'Something went wrong';
}
});
const stop = new FlowElement({
dataKey: 'stopElement',
processInternal: function (flowData) {
flowData.stop();
}
});
const neverRun = new FlowElement({
dataKey: 'neverRun',
processInternal: function (flowData) {
const data = new ElementDataDictionary({ flowElement: this, contents: { no: false } });
flowData.setElementData(data);
}
});
const syncPipeline = new PipelineBuilder().add(async).add(sync).add(error).add(stop).add(neverRun).build();
const syncFlowData = syncPipeline.createFlowData();
syncFlowData.evidence.add('header.user_agent', 'test');
syncFlowData.evidence.add('header.other', 'no');
syncFlowData.evidence.addObject({ test: 'testing' });
test('evidence add', () => {
expect(syncFlowData.evidence.get('header.user_agent')).toBe('test');
});
test('evidence addObject', () => {
expect(syncFlowData.evidence.get('test')).toBe('testing');
});
test('evidenceKeyFilter', () => {
const allEvidence = syncFlowData.evidence.getAll();
expect(Object.keys(sync.evidenceKeyFilter.filterEvidence(allEvidence))[0]).toBe('header.user_agent');
});
sync.properties = {
setup.sync.properties = {
integer: {

@@ -137,3 +47,3 @@ type: 'int',

test('Update property list on flowElement', done => {
sync.updateProperties().then(function () {
setup.sync.updateProperties().then(function () {
newPropertyFlowData.process().then(function (flowData) {

@@ -173,3 +83,3 @@ expect(flowData.getWhere('extra', 'test').integer).toBe(5);

syncFlowData.process().then(function () {
expect(syncFlowData.getFromElement(sync).get('integer')).toBe(5);
expect(syncFlowData.getFromElement(setup.sync).get('integer')).toBe(5);

@@ -180,2 +90,13 @@ done();

test('for of', done => {
syncFlowData.process().then(function () {
const keys = [];
for (const key of syncFlowData.sync) {
keys.push(key);
}
expect(JSON.stringify(keys)).toBe(JSON.stringify(['integer', 'boolean']));
done();
});
});
test('getWhere', done => {

@@ -207,6 +128,10 @@ syncFlowData.process().then(function () {

test('error data is populated', done => {
syncFlowData.process().then(function () {
expect(syncFlowData.errors.error[0]).toBe('Something went wrong');
const asyncPipeline = new PipelineBuilder()
.addParallel([setup.sync, setup.async]).build();
const asyncFlowData = asyncPipeline.createFlowData();
test('engines run in parallel', done => {
asyncFlowData.process().then(function () {
expect(asyncFlowData.get('sync').get('boolean')).toBe(false);
done();

@@ -216,11 +141,11 @@ });

let log;
const configPipeline = new PipelineBuilder()
.buildFromConfigurationFile(__dirname + '/config.json');
syncPipeline.on('error', function (error) {
log = error.message;
});
const configPipelineFlowData = configPipeline.createFlowData();
test('logging', done => {
syncFlowData.process().then(function () {
expect(log).toBe('Something went wrong');
test('build from config', done => {
configPipelineFlowData.process().then(function () {
expect(configPipelineFlowData.get('configTest')
.get('built')).toBe('hello_world');

@@ -231,31 +156,26 @@ done();

test('stop flag works', done => {
syncFlowData.process().then(function () {
expect(typeof syncFlowData.get('neverRun')).toBe('undefined');
test('aspectPropertyValue', done => {
const apvPipeline = new PipelineBuilder().add(setup.apvTest).build();
done();
});
});
const flowData = apvPipeline.createFlowData();
const asyncPipeline = new PipelineBuilder().addParallel([sync, async]).build();
const asyncFlowData = asyncPipeline.createFlowData();
flowData.process().then(function () {
expect(flowData.apv.get('yes').hasValue).toBe(true);
expect(flowData.apv.get('yes').value).toBe('success');
test('engines run in parallel', done => {
asyncFlowData.process().then(function () {
expect(asyncFlowData.get('sync').get('boolean')).toBe(false);
expect(flowData.apv.get('no').hasValue).toBe(false);
expect(expect(flowData.apv.no.noValueMessage).toBe('Value missing'));
done();
});
});
let error = '';
const configPipeline = new PipelineBuilder().buildFromConfigurationFile(__dirname + '/config.json');
try {
console.log(flowData.apv.no.value);
} catch (e) {
error = e;
}
const configPipelineFlowData = configPipeline.createFlowData();
expect(expect(error).toBe('Value missing'));
test('build from config', done => {
configPipelineFlowData.process().then(function () {
expect(configPipelineFlowData.get('configTest').get('built')).toBe('hello_world');
done();
});
});

17

tests/javascriptbuilder.test.js

@@ -53,3 +53,9 @@ const core = require('fiftyone.pipeline.core');

javascriptProperties: ['testengine.three'],
testengine: { one: 1, two: 2, three: "console.log('ok')", four: null, fournullreason: 'This property is not available' }
testengine: {
one: 1,
two: 2,
three: "console.log('ok')",
four: null,
fournullreason: 'This property is not available'
}
})

@@ -62,11 +68,2 @@ );

// test("javascript output", () =>
// {
// // Uncomment this to refresh the jsoutput.js file
// // fs.writeFileSync(__dirname + "/jsoutput.js", flowData.javascriptbuilder.javascript);
// expect(flowData.javascriptbuilder.javascript).toBe(fs.readFileSync(__dirname + "/jsoutput.js", "utf8"));
// });
test('sequence gives a session id', () => {

@@ -73,0 +70,0 @@ expect(flowData.evidence.get('query.session-id')).toBeTruthy();

examples/customFlowElements/1-simpleEvidenceFlowElement.js
examples/customFlowElements/2-clientSideEvidenceFlowElement.js
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc