@jupiterone/integration-sdk-runtime - npm Package Compare versions

Comparing version 4.0.1 to 4.1.0

dist/src/execution/dependencyGraph.d.ts

 import { DepGraph } from 'dependency-graph';
-import { Step, IntegrationStepResult, StepStartStates, ExecutionContext, StepExecutionContext, KeyNormalizationFunction } from '@jupiterone/integration-sdk-core';
+import { ExecutionContext, IntegrationStepResult, Step, StepExecutionContext, StepStartStates } from '@jupiterone/integration-sdk-core';
+import { GraphObjectStore } from '../storage';
+import { DuplicateKeyTracker } from './jobState';
 /**
@@ -22,2 +24,8 @@ * This function accepts a list of steps and constructs a dependency graph
  */
-export declare function executeStepDependencyGraph<TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext>(executionContext: TExecutionContext, inputGraph: DepGraph<Step<TStepExecutionContext>>, stepStartStates: StepStartStates, normalizeGraphObjectKey?: KeyNormalizationFunction): Promise<IntegrationStepResult[]>;
+export declare function executeStepDependencyGraph<TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext>({ executionContext, inputGraph, stepStartStates, duplicateKeyTracker, graphObjectStore, }: {
+    executionContext: TExecutionContext;
+    inputGraph: DepGraph<Step<TStepExecutionContext>>;
+    stepStartStates: StepStartStates;
+    duplicateKeyTracker: DuplicateKeyTracker;
+    graphObjectStore: GraphObjectStore;
+}): Promise<IntegrationStepResult[]>;

dist/src/execution/dependencyGraph.js

@@ -9,6 +9,5 @@ "use strict";
 const p_queue_1 = __importDefault(require("p-queue"));
-const storage_1 = require("../storage");
-const jobState_1 = require("./jobState");
 const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
 const metrics_1 = require("../metrics");
+const jobState_1 = require("./jobState");
 /**
@@ -51,3 +50,3 @@ * This function accepts a list of steps and constructs a dependency graph
  */
-function executeStepDependencyGraph(executionContext, inputGraph, stepStartStates, normalizeGraphObjectKey) {
+function executeStepDependencyGraph({ executionContext, inputGraph, stepStartStates, duplicateKeyTracker, graphObjectStore, }) {
     // create a clone of the dependencyGraph because mutating
@@ -62,4 +61,2 @@ // the input graph is icky
     const promiseQueue = new p_queue_1.default();
-    const duplicateKeyTracker = new jobState_1.DuplicateKeyTracker(normalizeGraphObjectKey);
-    const graphObjectStore = new storage_1.FileSystemGraphObjectStore();
     const stepResultsMap = buildStepResultsMap(inputGraph, stepStartStates);
@@ -70,3 +67,2 @@ const dataStore = new jobState_1.MemoryDataStore();
     }
-    integration_sdk_core_1.StepResultStatus;
     /**
@@ -129,3 +125,3 @@ * Updates the result of a step result with the provided satus
         const encounteredTypes = typeTracker.getEncounteredTypes();
-        const declaredTypesSet = new Set(getDeclaredTypesInStep(step));
+        const declaredTypesSet = new Set(getDeclaredTypesInStep(step).declaredTypes);
         const undeclaredTypes = encounteredTypes.filter((type) => !declaredTypesSet.has(type));
@@ -258,2 +254,3 @@ if (undeclaredTypes.length) {
             .filter((id) => stepStartStates[id].disabled).length > 0;
+        const { declaredTypes, partialTypes } = getDeclaredTypesInStep(step);
         return {
@@ -263,3 +260,4 @@ id: step.id,
             dependsOn: step.dependsOn,
-            declaredTypes: getDeclaredTypesInStep(step),
+            declaredTypes,
+            partialTypes,
             encounteredTypes: [],
@@ -275,8 +273,12 @@ status: stepStartStates[step.id].disabled || hasDisabledDependencies
 function getDeclaredTypesInStep(step) {
-    const declaredTypes = [
-        ...step.entities.map((e) => e._type),
-        ...step.relationships.map((e) => e._type),
-    ];
-    return declaredTypes;
+    const declaredTypes = [];
+    const partialTypes = [];
+    [...step.entities, ...step.relationships].map((e) => {
+        declaredTypes.push(e._type);
+        if (e.partial) {
+            partialTypes.push(e._type);
+        }
+    });
+    return { declaredTypes, partialTypes };
 }
 //# sourceMappingURL=dependencyGraph.js.map

dist/src/execution/executeIntegration.d.ts

@@ -1,2 +0,3 @@
-import { IntegrationInstance, IntegrationInvocationConfig, IntegrationStepResult, PartialDatasets, InvocationConfig, IntegrationLogger, ExecutionContext, StepExecutionContext } from '@jupiterone/integration-sdk-core';
+import { ExecutionContext, ExecutionHistory, IntegrationInstance, IntegrationInvocationConfig, IntegrationLogger, IntegrationStepResult, InvocationConfig, PartialDatasets, StepExecutionContext } from '@jupiterone/integration-sdk-core';
+import { GraphObjectStore } from '../storage';
 export interface ExecuteIntegrationResult {
@@ -10,3 +11,8 @@ integrationStepResults: IntegrationStepResult[];
     enableSchemaValidation?: boolean;
+    executionHistory?: ExecutionHistory;
+    graphObjectStore?: GraphObjectStore;
 }
+interface ExecuteWithContextOptions {
+    graphObjectStore?: GraphObjectStore;
+}
 /**
@@ -16,9 +22,7 @@ * Starts execution of an integration instance generated from local environment
  */
-export declare function executeIntegrationLocally(config: IntegrationInvocationConfig, options?: {
-    enableSchemaValidation?: boolean;
-}): Promise<ExecuteIntegrationResult>;
+export declare function executeIntegrationLocally(config: IntegrationInvocationConfig, options?: ExecuteIntegrationOptions): Promise<ExecuteIntegrationResult>;
 /**
  * Starts execution of an integration instance.
  */
-export declare function executeIntegrationInstance(logger: IntegrationLogger | IntegrationLogger, instance: IntegrationInstance, config: IntegrationInvocationConfig, options?: ExecuteIntegrationOptions): Promise<ExecuteIntegrationResult>;
+export declare function executeIntegrationInstance(logger: IntegrationLogger, instance: IntegrationInstance, config: IntegrationInvocationConfig, options?: ExecuteIntegrationOptions): Promise<ExecuteIntegrationResult>;
 /**
@@ -28,3 +32,3 @@ * Executes an integration and performs actions defined by the config
  */
-export declare function executeWithContext<TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext>(context: TExecutionContext, config: InvocationConfig<TExecutionContext, TStepExecutionContext>): Promise<ExecuteIntegrationResult>;
+export declare function executeWithContext<TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext>(context: TExecutionContext, config: InvocationConfig<TExecutionContext, TStepExecutionContext>, options?: ExecuteWithContextOptions): Promise<ExecuteIntegrationResult>;
 export {};

dist/src/execution/executeIntegration.js

@@ -5,7 +5,9 @@ "use strict";
 const fileSystem_1 = require("../fileSystem");
+const logger_1 = require("../logger");
+const metrics_1 = require("../metrics");
+const storage_1 = require("../storage");
 const instance_1 = require("./instance");
-const logger_1 = require("../logger");
+const jobState_1 = require("./jobState");
 const step_1 = require("./step");
 const validation_1 = require("./validation");
-const metrics_1 = require("../metrics");
 /**
@@ -31,3 +33,3 @@ * Starts execution of an integration instance generated from local environment
 async function executeIntegrationInstance(logger, instance, config, options = {}) {
-    if (options.enableSchemaValidation) {
+    if (options.enableSchemaValidation === true) {
         process.env.ENABLE_GRAPH_OBJECT_SCHEMA_VALIDATION = 'true';
@@ -41,3 +43,6 @@ }
             logger,
-        }, config),
+            history: options.executionHistory,
+        }, config, {
+            graphObjectStore: options.graphObjectStore,
+        }),
     });
@@ -50,3 +55,3 @@ }
  */
-async function executeWithContext(context, config) {
+async function executeWithContext(context, config, options = {}) {
     var _a, _b, _c;
@@ -64,3 +69,10 @@ await fileSystem_1.removeStorageDirectory();
     validation_1.validateStepStartStates(config.integrationSteps, stepStartStates);
-    const integrationStepResults = await step_1.executeSteps(context, config.integrationSteps, stepStartStates, config.normalizeGraphObjectKey);
+    const { graphObjectStore = new storage_1.FileSystemGraphObjectStore() } = options;
+    const integrationStepResults = await step_1.executeSteps({
+        executionContext: context,
+        integrationSteps: config.integrationSteps,
+        stepStartStates,
+        duplicateKeyTracker: new jobState_1.DuplicateKeyTracker(config.normalizeGraphObjectKey),
+        graphObjectStore,
+    });
     const partialDatasets = step_1.determinePartialDatasetsFromStepExecutionResults(integrationStepResults);
@@ -67,0 +79,0 @@ const summary = {

dist/src/execution/jobState.d.ts

 import { JobState, KeyNormalizationFunction } from '@jupiterone/integration-sdk-core';
-import { FileSystemGraphObjectStore } from '../storage';
+import { GraphObjectStore } from '../storage';
 export interface DuplicateKeyTrackerGraphObjectMetadata {
@@ -35,5 +35,5 @@ _type: string;
     typeTracker: TypeTracker;
-    graphObjectStore: FileSystemGraphObjectStore;
+    graphObjectStore: GraphObjectStore;
     dataStore: MemoryDataStore;
 }
 export declare function createStepJobState({ stepId, duplicateKeyTracker, typeTracker, graphObjectStore, dataStore, }: CreateStepJobStateParams): JobState;

dist/src/execution/step.d.ts

@@ -1,3 +0,11 @@
-import { InvocationConfig, ExecutionContext, StepExecutionContext, IntegrationStepResult, StepStartStates, PartialDatasets, Step, KeyNormalizationFunction } from '@jupiterone/integration-sdk-core';
-export declare function executeSteps<TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext>(context: TExecutionContext, integrationSteps: Step<TStepExecutionContext>[], stepStartStates: StepStartStates, normalizeGraphObjectKey?: KeyNormalizationFunction): Promise<IntegrationStepResult[]>;
+import { ExecutionContext, IntegrationStepResult, InvocationConfig, PartialDatasets, Step, StepExecutionContext, StepStartStates } from '@jupiterone/integration-sdk-core';
+import { GraphObjectStore } from '../storage';
+import { DuplicateKeyTracker } from './jobState';
+export declare function executeSteps<TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext>({ executionContext, integrationSteps, stepStartStates, duplicateKeyTracker, graphObjectStore, }: {
+    executionContext: TExecutionContext;
+    integrationSteps: Step<TStepExecutionContext>[];
+    stepStartStates: StepStartStates;
+    duplicateKeyTracker: DuplicateKeyTracker;
+    graphObjectStore: GraphObjectStore;
+}): Promise<IntegrationStepResult[]>;
 export declare function getDefaultStepStartStates<TStepExecutionContext extends StepExecutionContext>(steps: Step<TStepExecutionContext>[]): StepStartStates;
@@ -4,0 +12,0 @@ export declare function determinePartialDatasetsFromStepExecutionResults(stepResults: IntegrationStepResult[]): PartialDatasets;

dist/src/execution/step.js

@@ -8,7 +8,12 @@ "use strict";
 const uniq_1 = __importDefault(require("lodash/uniq"));
+const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
 const dependencyGraph_1 = require("./dependencyGraph");
-const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
-async function executeSteps(context, integrationSteps, stepStartStates, normalizeGraphObjectKey) {
-    const stepGraph = dependencyGraph_1.buildStepDependencyGraph(integrationSteps);
-    return dependencyGraph_1.executeStepDependencyGraph(context, stepGraph, stepStartStates, normalizeGraphObjectKey);
+async function executeSteps({ executionContext, integrationSteps, stepStartStates, duplicateKeyTracker, graphObjectStore, }) {
+    return dependencyGraph_1.executeStepDependencyGraph({
+        executionContext,
+        inputGraph: dependencyGraph_1.buildStepDependencyGraph(integrationSteps),
+        stepStartStates,
+        duplicateKeyTracker,
+        graphObjectStore,
+    });
 }
@@ -27,8 +32,16 @@ exports.executeSteps = executeSteps;
     return stepResults.reduce((partialDatasets, stepResult) => {
+        const stepPartialDatasets = {
+            types: [],
+        };
+        if (stepResult.status !== integration_sdk_core_1.StepResultStatus.DISABLED) {
+            stepPartialDatasets.types.push(...stepResult.partialTypes);
+        }
         if (stepResult.status === integration_sdk_core_1.StepResultStatus.FAILURE ||
             stepResult.status ===
                 integration_sdk_core_1.StepResultStatus.PARTIAL_SUCCESS_DUE_TO_DEPENDENCY_FAILURE) {
-            partialDatasets.types = uniq_1.default(partialDatasets.types.concat(stepResult.declaredTypes));
+            stepPartialDatasets.types.push(...stepResult.declaredTypes);
         }
-        return partialDatasets;
+        return {
+            types: uniq_1.default(partialDatasets.types.concat(stepPartialDatasets.types)),
+        };
     }, { types: [] });
@@ -35,0 +48,0 @@ }

dist/src/index.d.ts

@@ -7,1 +7,2 @@ export * from './api';
 export * from './metrics';
+export * from './storage';

dist/src/index.js

@@ -19,2 +19,3 @@ "use strict";
 __exportStar(require("./metrics"), exports);
+__exportStar(require("./storage"), exports);
 //# sourceMappingURL=index.js.map

dist/src/storage/FileSystemGraphObjectStore/BucketMap.d.ts

@@ -9,3 +9,4 @@ /**
 export declare class BucketMap<T = any> extends Map<string, T[]> {
-    totalItemCount: number;
+    private internalTotalItemCount;
+    get totalItemCount(): number;
     add(key: string, values: T[]): void;
@@ -12,0 +13,0 @@ set(key: string, values: T[]): this;

dist/src/storage/FileSystemGraphObjectStore/BucketMap.js

@@ -14,4 +14,7 @@ "use strict";
         super(...arguments);
-        this.totalItemCount = 0;
+        this.internalTotalItemCount = 0;
     }
+    get totalItemCount() {
+        return this.internalTotalItemCount;
+    }
     add(key, values) {
@@ -25,3 +28,3 @@ const existingValues = this.get(key);
         super.set(key, values);
-        this.totalItemCount += values.length - existingItemCount;
+        this.internalTotalItemCount += values.length - existingItemCount;
         return this;
@@ -33,3 +36,3 @@ }
         const deleteResult = super.delete(key);
-        this.totalItemCount -= existingItemCount;
+        this.internalTotalItemCount -= existingItemCount;
         return deleteResult;
@@ -36,0 +39,0 @@ }

dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.d.ts

 import { Sema } from 'async-sema';
-import { Entity, Relationship, GraphObjectFilter, GraphObjectIteratee, GraphObjectLookupKey } from '@jupiterone/integration-sdk-core';
+import { Entity, GraphObjectFilter, GraphObjectIteratee, GraphObjectLookupKey, Relationship, GraphObjectStore } from '@jupiterone/integration-sdk-core';
 import { BucketMap } from './BucketMap';
 export declare const GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
-export declare class FileSystemGraphObjectStore {
+export declare class FileSystemGraphObjectStore implements GraphObjectStore {
     semaphore: Sema;
@@ -7,0 +7,0 @@ entityStorageMap: BucketMap<Entity>;

dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js

@@ -7,7 +7,7 @@ "use strict";
 exports.FileSystemGraphObjectStore = exports.GRAPH_OBJECT_BUFFER_THRESHOLD = void 0;
+const async_sema_1 = require("async-sema");
 const p_map_1 = __importDefault(require("p-map"));
-const async_sema_1 = require("async-sema");
 const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
+const BucketMap_1 = require("./BucketMap");
 const flushDataToDisk_1 = require("./flushDataToDisk");
-const BucketMap_1 = require("./BucketMap");
 const indices_1 = require("./indices");
@@ -14,0 +14,0 @@ exports.GRAPH_OBJECT_BUFFER_THRESHOLD = 500; // arbitrarily selected, subject to tuning

dist/src/storage/FileSystemGraphObjectStore/index.d.ts

 export * from './FileSystemGraphObjectStore';
+export * from './BucketMap';

dist/src/storage/FileSystemGraphObjectStore/index.js

@@ -14,2 +14,3 @@ "use strict";
 __exportStar(require("./FileSystemGraphObjectStore"), exports);
+__exportStar(require("./BucketMap"), exports);
 //# sourceMappingURL=index.js.map

dist/src/storage/index.d.ts

@@ -0,1 +1,2 @@
+export { GraphObjectStore } from '@jupiterone/integration-sdk-core';
 export * from './FileSystemGraphObjectStore';

package.json

 {
   "name": "@jupiterone/integration-sdk-runtime",
-  "version": "4.0.1",
+  "version": "4.1.0",
   "description": "The SDK for developing JupiterOne integrations",
@@ -26,3 +26,3 @@ "main": "dist/src/index.js",
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^4.0.1",
+    "@jupiterone/integration-sdk-core": "^4.1.0",
     "@lifeomic/alpha": "^1.1.3",
@@ -45,3 +45,3 @@ "async-sema": "^3.1.0",
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^4.0.1",
+    "@jupiterone/integration-sdk-private-test-utils": "^4.1.0",
     "@types/uuid": "^7.0.2",
@@ -52,3 +52,3 @@ "get-port": "^5.1.1",
   },
-  "gitHead": "93c9e694cf1fe11dd89aa2e82164571459892aca"
+  "gitHead": "144834ddf08b540e071423c01c478864c19f500f"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

445486

increased by1.3%

Number of package files

85

increased by3.66%

Lines of code

2301

increased by3.09%

Dependency changes

• Updated@jupiterone/integration-sdk-core@^4.1.0