@grouparoo/app-templates - npm Package Compare versions

Comparing version 0.8.1-alpha.0 to 0.8.1-alpha.1

dist/source/query/meta.d.ts

@@ -1,2 +0,2 @@
-import { PluginConnection, ExecuteQueryMethod, ValidateQueryMethod, GetChangedRowsMethod } from "./options";
+import { PluginConnection, ExecuteQueryMethod, ValidateQueryMethod, GetRowsMethod } from "./options";
 import { GetTablesMethod } from "../table";
@@ -15,3 +15,3 @@ import { ConnectionOptionsOption } from "@grouparoo/core";
         validateQuery?: ValidateQueryMethod;
-        getChangedRows?: GetChangedRowsMethod;
+        getRows?: GetRowsMethod;
         getTables?: GetTablesMethod;
@@ -18,0 +18,0 @@ }): PluginConnection;

dist/source/query/meta.js

@@ -5,3 +5,3 @@ "use strict";
 const options_1 = require("./options");
-const buildConnection = ({ name, displayName, description, apps, tableOptionDescription = null, options = [], executeQuery, getChangedRows, validateQuery = options_1.validateGenericQuery, getTables, }) => {
+const buildConnection = ({ name, displayName, description, apps, tableOptionDescription = null, options = [], executeQuery, getRows, validateQuery = options_1.validateGenericQuery, getTables, }) => {
     const sourceOptions = (0, options_1.getSourceOptions)({ getTables });
@@ -14,4 +14,4 @@ const recordProperty = (0, options_1.getRecordProperty)({
     const scheduleOptions = async (args) => (0, options_1.getScheduleOptions)(args);
-    // const scheduleOptions = getChangedRows ? getScheduleOptions() : null;
-    const records = getChangedRows ? (0, options_1.getRecordsMethod)(getChangedRows) : null;
+    // const scheduleOptions = getRows ? getScheduleOptions() : null;
+    const importRecords = getRows ? (0, options_1.getRecordsMethod)(getRows) : null;
     const plugin = {
@@ -22,2 +22,3 @@ name,
         description,
+        supportIncrementalSchedule: false,
         apps,
@@ -30,3 +31,3 @@ options,
             recordProperty,
-            records,
+            importRecords,
             propertyOptions,
@@ -33,0 +34,0 @@ scheduleOptions,

dist/source/query/records.d.ts

 import { SimpleAppOptions, SimpleScheduleOptions, RecordsPluginMethod, SimpleSourceOptions } from "@grouparoo/core";
 import { DataResponseRow } from "../shared/types";
-export interface GetChangedRowsMethod<ConnectionType = any> {
+export interface GetRowsMethod<ConnectionType = any> {
     (argument: {
@@ -12,4 +12,5 @@ connection: ConnectionType;
         sourceOptions?: SimpleSourceOptions;
+        incremental: boolean;
     }): Promise<DataResponseRow[]>;
 }
-export declare const getRecordsMethod: (getChangedRows: GetChangedRowsMethod) => RecordsPluginMethod<any>;
+export declare const getRecordsMethod: (getRows: GetRowsMethod) => RecordsPluginMethod<any>;

dist/source/query/records.js

@@ -5,12 +5,6 @@ "use strict";
 const core_1 = require("@grouparoo/core");
-const actionhero_1 = require("actionhero");
-const getRecordsMethod = (getChangedRows) => {
-    const records = async ({ scheduleOptions, connection, highWaterMark, run, appId, appOptions, properties, sourceOptions, }) => {
-        let offset = highWaterMark.offset
-            ? parseInt(highWaterMark.offset.toString())
-            : 0;
-        const limit = highWaterMark.limit
-            ? parseInt(highWaterMark.limit.toString())
-            : actionhero_1.config.batchSize.imports;
-        const rows = await getChangedRows({
+const getRecordsMethod = (getRows) => {
+    const records = async ({ scheduleOptions, connection, run, appId, appOptions, properties, sourceOptions, schedule, sourceOffset, limit, }) => {
+        const offset = parseInt(String(sourceOffset !== null && sourceOffset !== void 0 ? sourceOffset : 0));
+        const rows = await getRows({
             appId,
@@ -23,2 +17,3 @@ appOptions,
             connection,
+            incremental: schedule.incremental,
         });
@@ -34,9 +29,7 @@ const property = properties.find((p) => p.id === scheduleOptions.propertyId);
         }
-        else {
-            offset = 0;
-        }
+        const nextOffset = rows.length > 0 ? offset + rows.length : 0;
         return {
             importsCount: rows.length,
-            highWaterMark: { offset: offset + rows.length, limit },
-            sourceOffset: null,
+            highWaterMark: { offset: nextOffset, limit },
+            sourceOffset: nextOffset,
         };
@@ -43,0 +36,0 @@ };

dist/source/table/getExamples.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getSortableColumnExamples = exports.getColumnExamples = exports.getExampleRows = void 0;
-const core_1 = require("@grouparoo/core");
 const getExampleRows = async ({ connection, appOptions, sourceOptions, appId, tableName, columns, getSampleRows, getColumns, }) => {
@@ -76,6 +75,3 @@ let rows = await getSampleRows({
     Object.keys(columns).map((colName) => {
-        const { filterOperations } = columns[colName];
-        if (filterOperations.includes(core_1.FilterOperation.GreaterThanOrEqual)) {
-            choices.push(colName);
-        }
+        choices.push(colName);
     });
@@ -82,0 +78,0 @@ return choices.map((col) => {

dist/source/table/meta.d.ts

@@ -1,2 +0,2 @@
-import { GetSampleRowsMethod, GetTablesMethod, GetColumnDefinitionsMethod, PluginConnection, GetChangedRowsMethod, GetPropertyValueMethod, GetPropertyValuesMethod, GetChangedRowCountMethod, SourceOptionsExtra } from "./options";
+import { GetSampleRowsMethod, GetTablesMethod, GetColumnDefinitionsMethod, PluginConnection, GetRowsMethod, GetPropertyValueMethod, GetPropertyValuesMethod, GetRowCountMethod, SourceOptionsExtra } from "./options";
 export interface BuildConnectionMethod {
@@ -14,8 +14,8 @@ (argument: {
         getTables: GetTablesMethod;
-        getChangedRows: GetChangedRowsMethod;
+        getRows: GetRowsMethod;
         getPropertyValue: GetPropertyValueMethod;
         getPropertyValues?: GetPropertyValuesMethod;
-        getChangedRowCount: GetChangedRowCountMethod;
+        getRowCount: GetRowCountMethod;
     }): PluginConnection;
 }
 export declare const buildConnection: BuildConnectionMethod;

dist/source/table/meta.js

@@ -5,3 +5,3 @@ "use strict";
 const options_1 = require("./options");
-const buildConnection = ({ name, displayName, description, apps, tableOptionDescription = null, tableOptionDisplayName = null, sourceOptions: additionalOptions, getSampleRows, getColumns, getTables, getChangedRows, getPropertyValue, getPropertyValues, getChangedRowCount, }) => {
+const buildConnection = ({ name, displayName, description, apps, tableOptionDescription = null, tableOptionDisplayName = null, sourceOptions: additionalOptions, getSampleRows, getColumns, getTables, getRows, getPropertyValue, getPropertyValues, getRowCount, }) => {
     const propertyOptions = async (args) => (0, options_1.getPropertyOptions)(args, {
@@ -27,4 +27,4 @@ getSampleRows,
     const uniquePropertyBootstrapOptions = (0, options_1.getUniquePropertyBootstrapOptions)();
-    const records = (0, options_1.getRecords)({
-        getChangedRows,
+    const importRecords = (0, options_1.getRecords)({
+        getRows,
     });
@@ -42,3 +42,3 @@ const recordProperty = getPropertyValue
     const sourceRunPercentComplete = (0, options_1.getSourceRunPercentComplete)({
-        getChangedRowCount,
+        getRowCount,
     });
@@ -56,2 +56,3 @@ const options = ((additionalOptions === null || additionalOptions === void 0 ? void 0 : additionalOptions.options) || []).concat({
         description,
+        supportIncrementalSchedule: true,
         apps,
@@ -67,3 +68,3 @@ options,
             uniquePropertyBootstrapOptions,
-            records,
+            importRecords,
             recordProperty,
@@ -70,0 +71,0 @@ recordProperties,

dist/source/table/pluginMethods.d.ts

@@ -95,3 +95,3 @@ import { ConnectionOptionsOption, PropertyTypes, SimpleAppOptions, SimpleSourceOptions, FilterOperation, AggregationMethod } from "@grouparoo/core";
 }
-export interface GetChangedRowsMethod<AppConnection = any> {
+export interface GetRowsMethod<AppConnection = any> {
     (argument: {
@@ -109,6 +109,7 @@ connection: AppConnection;
         matchConditions: MatchCondition[];
+        incremental: boolean;
         highWaterMarkKey: string;
     }): Promise<DataResponseRow[]>;
 }
-export interface GetChangedRowCountMethod<AppConnection = any> {
+export interface GetRowCountMethod<AppConnection = any> {
     (argument: {
@@ -119,2 +120,3 @@ connection: AppConnection;
         appId: string;
+        incremental: boolean;
         tableName: string;
@@ -121,0 +123,0 @@ highWaterMarkCondition: MatchCondition;

dist/source/table/records.d.ts

@@ -1,6 +0,6 @@
-import { MatchCondition, GetChangedRowsMethod } from "./pluginMethods";
+import { MatchCondition, GetRowsMethod } from "./pluginMethods";
 import { RecordsPluginMethod, Run, Source, HighWaterMark } from "@grouparoo/core";
 export interface GetRecordsMethod {
     (argument: {
-        getChangedRows: GetChangedRowsMethod;
+        getRows: GetRowsMethod;
     }): RecordsPluginMethod;
@@ -15,3 +15,3 @@ }
     (argument: {
-        getChangedRows: GetChangedRowsMethod;
+        getRows: GetRowsMethod;
     }): RecordsPluginMethod;
@@ -18,0 +18,0 @@ }

dist/source/table/records.js

@@ -7,4 +7,4 @@ "use strict";
 const core_1 = require("@grouparoo/core");
-const getRecords = ({ getChangedRows }) => {
-    const records = async ({ connection, appOptions, appId, run, sourceMapping, scheduleOptions, source, limit, highWaterMark, sourceOffset, scheduleFilters, }) => {
+const getRecords = ({ getRows }) => {
+    const records = async ({ connection, appOptions, appId, run, sourceMapping, scheduleOptions, source, limit, highWaterMark, sourceOffset, scheduleFilters, schedule, }) => {
         var _a;
@@ -29,3 +29,3 @@ const { tableName, highWaterMarkCondition } = await (0, exports.getChangeVariables)({
         }
-        const results = await getChangedRows({
+        const results = await getRows({
             connection,
@@ -42,2 +42,3 @@ appOptions,
             matchConditions,
+            incremental: schedule.incremental,
         });
@@ -53,5 +54,9 @@ let nextSourceOffset = 0;
             const newValue = lastRow[highWaterMarkKey].toString();
-            if (currentValue && newValue === currentValue) {
+            if (!schedule.incremental) {
                 nextSourceOffset = sourceOffset + limit;
+                nextHighWaterMark[highWaterMarkAndSortColumnASC] = newValue;
             }
+            else if (currentValue && newValue === currentValue) {
+                nextSourceOffset = sourceOffset + limit;
+            }
             else {
@@ -58,0 +63,0 @@ nextHighWaterMark[highWaterMarkAndSortColumnASC] = newValue;

dist/source/table/scheduleOptions.js

@@ -11,3 +11,3 @@ "use strict";
             required: true,
-            description: "which column to scan for changes",
+            description: "the column used to sort your data (ORDER BY, ASC)",
             type: "list",
@@ -14,0 +14,0 @@ options: async ({ connection, appOptions, appId, sourceOptions }) => {

dist/source/table/sourceRunPercentComplete.d.ts

 import { SourceRunPercentCompleteMethod } from "@grouparoo/core";
-import { GetChangedRowCountMethod } from "./pluginMethods";
+import { GetRowCountMethod } from "./pluginMethods";
 export interface GetSourceRunPercentCompleteMethod {
     (argument: {
-        getChangedRowCount: GetChangedRowCountMethod;
+        getRowCount: GetRowCountMethod;
     }): SourceRunPercentCompleteMethod;
 }
 export declare const getSourceRunPercentComplete: GetSourceRunPercentCompleteMethod;

dist/source/table/sourceRunPercentComplete.js

@@ -6,4 +6,4 @@ "use strict";
 const records_1 = require("./records");
-const getSourceRunPercentComplete = ({ getChangedRowCount, }) => {
-    const sourceRunPercentComplete = async ({ connection, appOptions, appId, source, highWaterMark, run, scheduleFilters, }) => {
+const getSourceRunPercentComplete = ({ getRowCount, }) => {
+    const sourceRunPercentComplete = async ({ connection, appOptions, appId, source, schedule, highWaterMark, run, scheduleFilters, }) => {
         const { tableName, highWaterMarkCondition } = await (0, records_1.getChangeVariables)({
@@ -23,3 +23,3 @@ run,
         }
-        const count = await getChangedRowCount({
+        const count = await getRowCount({
             connection,
@@ -31,2 +31,3 @@ appOptions,
             highWaterMarkCondition,
+            incremental: schedule.incremental,
         });
@@ -33,0 +34,0 @@ const percentComplete = count > 0 ? Math.floor((run.importsCreated / count) * 100) : 100;

package.json

@@ -5,3 +5,3 @@ {
   "description": "Common patterns for Grouparoo plugins that contribute apps",
-  "version": "0.8.1-alpha.0",
+  "version": "0.8.1-alpha.1",
   "license": "MPL-2.0",
@@ -32,16 +32,16 @@ "private": false,
   "devDependencies": {
-    "@grouparoo/core": "0.8.1-alpha.0",
-    "@grouparoo/spec-helper": "0.8.1-alpha.0",
+    "@grouparoo/core": "0.8.1-alpha.1",
+    "@grouparoo/spec-helper": "0.8.1-alpha.1",
     "@types/jest": "*",
     "@types/node": "16.*.*",
     "@types/uuid": "*",
-    "actionhero": "28.1.4",
+    "actionhero": "28.1.7",
     "jest": "27.4.5",
-    "nock": "13.2.1",
+    "nock": "13.2.2",
     "prettier": "2.5.1",
-    "ts-jest": "27.1.2",
-    "typescript": "4.5.4",
+    "ts-jest": "27.1.3",
+    "typescript": "4.5.5",
     "uuid": "8.3.2"
   },
-  "gitHead": "92cbb610b57907d937b1fe802a186ef4a9732dc9"
+  "gitHead": "a95f9a45dbb6708ede0fabcc26939cff871299f1"
 }

src/source/query/meta.ts

@@ -15,3 +15,3 @@ import {
   validateGenericQuery,
-  GetChangedRowsMethod,
+  GetRowsMethod,
 } from "./options";
@@ -32,3 +32,3 @@ import { GetTablesMethod } from "../table";
     validateQuery?: ValidateQueryMethod;
-    getChangedRows?: GetChangedRowsMethod;
+    getRows?: GetRowsMethod;
     getTables?: GetTablesMethod;
@@ -46,3 +46,3 @@ }): PluginConnection;
   executeQuery,
-  getChangedRows,
+  getRows,
   validateQuery = validateGenericQuery,
@@ -62,4 +62,4 @@ getTables,
 
-  // const scheduleOptions = getChangedRows ? getScheduleOptions() : null;
-  const records = getChangedRows ? getRecordsMethod(getChangedRows) : null;
+  // const scheduleOptions = getRows ? getScheduleOptions() : null;
+  const importRecords = getRows ? getRecordsMethod(getRows) : null;
 
@@ -71,2 +71,3 @@ const plugin: PluginConnection = {
     description,
+    supportIncrementalSchedule: false,
     apps,
@@ -79,3 +80,3 @@ options,
       recordProperty,
-      records,
+      importRecords,
       propertyOptions,
@@ -82,0 +83,0 @@ scheduleOptions,

src/source/query/records.ts

@@ -8,6 +8,5 @@ import {
 } from "@grouparoo/core";
-import { config } from "actionhero";
 import { DataResponseRow } from "../shared/types";
 
-export interface GetChangedRowsMethod<ConnectionType = any> {
+export interface GetRowsMethod<ConnectionType = any> {
   (argument: {
@@ -21,10 +20,10 @@ connection: ConnectionType;
     sourceOptions?: SimpleSourceOptions;
+    incremental: boolean;
   }): Promise<DataResponseRow[]>;
 }
 
-export const getRecordsMethod = (getChangedRows: GetChangedRowsMethod) => {
+export const getRecordsMethod = (getRows: GetRowsMethod) => {
   const records: RecordsPluginMethod = async ({
     scheduleOptions,
     connection,
-    highWaterMark,
     run,
@@ -35,11 +34,8 @@ appId,
     sourceOptions,
+    schedule,
+    sourceOffset,
+    limit,
   }) => {
-    let offset = highWaterMark.offset
-      ? parseInt(highWaterMark.offset.toString())
-      : 0;
-    const limit = highWaterMark.limit
-      ? parseInt(highWaterMark.limit.toString())
-      : config.batchSize.imports;
-
-    const rows = await getChangedRows({
+    const offset = parseInt(String(sourceOffset ?? 0));
+    const rows = await getRows({
       appId,
@@ -52,2 +48,3 @@ appOptions,
       connection,
+      incremental: schedule.incremental,
     });
@@ -67,10 +64,9 @@
       await plugin.createImports(propertyMapping, run, rows);
-    } else {
-      offset = 0;
     }
 
+    const nextOffset = rows.length > 0 ? offset + rows.length : 0;
     return {
       importsCount: rows.length,
-      highWaterMark: { offset: offset + rows.length, limit },
-      sourceOffset: null,
+      highWaterMark: { offset: nextOffset, limit },
+      sourceOffset: nextOffset,
     };
@@ -77,0 +73,0 @@ };

src/source/table/getExamples.ts

@@ -140,6 +140,3 @@ import {
   Object.keys(columns).map((colName) => {
-    const { filterOperations } = columns[colName];
-    if (filterOperations.includes(FilterOperation.GreaterThanOrEqual)) {
-      choices.push(colName);
-    }
+    choices.push(colName);
   });
@@ -146,0 +143,0 @@

src/source/table/meta.ts

@@ -21,3 +21,3 @@ import {
   getRecords,
-  GetChangedRowsMethod,
+  GetRowsMethod,
   RecordPropertyPluginMethod,
@@ -31,3 +31,3 @@ RecordPropertiesPluginMethod,
   getSourceRunPercentComplete,
-  GetChangedRowCountMethod,
+  GetRowCountMethod,
   SourceOptionsExtra,
@@ -49,6 +49,6 @@ AggregationMethod,
     getTables: GetTablesMethod;
-    getChangedRows: GetChangedRowsMethod;
+    getRows: GetRowsMethod;
     getPropertyValue: GetPropertyValueMethod;
     getPropertyValues?: GetPropertyValuesMethod;
-    getChangedRowCount: GetChangedRowCountMethod;
+    getRowCount: GetRowCountMethod;
   }): PluginConnection;
@@ -68,6 +68,6 @@ }
   getTables,
-  getChangedRows,
+  getRows,
   getPropertyValue,
   getPropertyValues,
-  getChangedRowCount,
+  getRowCount,
 }) => {
@@ -97,4 +97,4 @@ const propertyOptions: PropertyOptionsMethod = async (args) =>
     getUniquePropertyBootstrapOptions();
-  const records: RecordsPluginMethod = getRecords({
-    getChangedRows,
+  const importRecords: RecordsPluginMethod = getRecords({
+    getRows,
   });
@@ -113,3 +113,3 @@ const recordProperty: RecordPropertyPluginMethod = getPropertyValue
     getSourceRunPercentComplete({
-      getChangedRowCount,
+      getRowCount,
     });
@@ -129,2 +129,3 @@
     description,
+    supportIncrementalSchedule: true,
     apps,
@@ -140,3 +141,3 @@ options,
       uniquePropertyBootstrapOptions,
-      records,
+      importRecords,
       recordProperty,
@@ -143,0 +144,0 @@ recordProperties,

src/source/table/pluginMethods.ts

@@ -112,3 +112,3 @@ import {
 }
-export interface GetChangedRowsMethod<AppConnection = any> {
+export interface GetRowsMethod<AppConnection = any> {
   (argument: {
@@ -126,2 +126,3 @@ connection: AppConnection;
     matchConditions: MatchCondition[];
+    incremental: boolean;
     highWaterMarkKey: string; // for the result in the returned row
@@ -131,3 +132,3 @@ }): Promise<DataResponseRow[]>;
 
-export interface GetChangedRowCountMethod<AppConnection = any> {
+export interface GetRowCountMethod<AppConnection = any> {
   (argument: {
@@ -138,2 +139,3 @@ connection: AppConnection;
     appId: string;
+    incremental: boolean;
     tableName: string;
@@ -140,0 +142,0 @@ highWaterMarkCondition: MatchCondition;

src/source/table/records.ts

@@ -5,3 +5,3 @@ import {
   MatchCondition,
-  GetChangedRowsMethod,
+  GetRowsMethod,
 } from "./pluginMethods";
@@ -19,6 +19,6 @@ import { getFilterOperation } from "./getFilterOperation";
 export interface GetRecordsMethod {
-  (argument: { getChangedRows: GetChangedRowsMethod }): RecordsPluginMethod;
+  (argument: { getRows: GetRowsMethod }): RecordsPluginMethod;
 }
 
-export const getRecords: GetRecordsMethod = ({ getChangedRows }) => {
+export const getRecords: GetRecordsMethod = ({ getRows }) => {
   const records: RecordsPluginMethod = async ({
@@ -36,2 +36,3 @@ connection,
     scheduleFilters,
+    schedule,
   }) => {
@@ -60,3 +61,3 @@ const { tableName, highWaterMarkCondition } = await getChangeVariables({
 
-    const results = await getChangedRows({
+    const results = await getRows({
       connection,
@@ -73,2 +74,3 @@ appOptions,
       matchConditions,
+      incremental: schedule.incremental,
     });
@@ -90,4 +92,7 @@
 
-      if (currentValue && newValue === currentValue) {
+      if (!schedule.incremental) {
         nextSourceOffset = sourceOffset + limit;
+        nextHighWaterMark[highWaterMarkAndSortColumnASC] = newValue;
+      } else if (currentValue && newValue === currentValue) {
+        nextSourceOffset = sourceOffset + limit;
       } else {
@@ -115,3 +120,3 @@ nextHighWaterMark[highWaterMarkAndSortColumnASC] = newValue;
 export interface Get {
-  (argument: { getChangedRows: GetChangedRowsMethod }): RecordsPluginMethod;
+  (argument: { getRows: GetRowsMethod }): RecordsPluginMethod;
 }
@@ -118,0 +123,0 @@ export interface GetChangeVariablesMethod {

src/source/table/scheduleOptions.ts

@@ -29,3 +29,3 @@ import {
       required: true,
-      description: "which column to scan for changes",
+      description: "the column used to sort your data (ORDER BY, ASC)",
       type: "list",
@@ -32,0 +32,0 @@ options: async ({ connection, appOptions, appId, sourceOptions }) => {

src/source/table/sourceRunPercentComplete.ts

 import { SourceRunPercentCompleteMethod } from "@grouparoo/core";
 import { getFilterOperation } from "./getFilterOperation";
-import { GetChangedRowCountMethod, MatchCondition } from "./pluginMethods";
+import { GetRowCountMethod, MatchCondition } from "./pluginMethods";
 import { getChangeVariables } from "./records";
@@ -8,3 +8,3 @@
   (argument: {
-    getChangedRowCount: GetChangedRowCountMethod;
+    getRowCount: GetRowCountMethod;
   }): SourceRunPercentCompleteMethod;
@@ -14,3 +14,3 @@ }
 export const getSourceRunPercentComplete: GetSourceRunPercentCompleteMethod = ({
-  getChangedRowCount,
+  getRowCount,
 }) => {
@@ -22,2 +22,3 @@ const sourceRunPercentComplete: SourceRunPercentCompleteMethod = async ({
     source,
+    schedule,
     highWaterMark,
@@ -44,3 +45,3 @@ run,
 
-    const count = await getChangedRowCount({
+    const count = await getRowCount({
       connection,
@@ -52,2 +53,3 @@ appOptions,
       highWaterMarkCondition,
+      incremental: schedule.incremental,
     });
@@ -54,0 +56,0 @@

No alert changes

Improved metrics

Lines of code

6514

increased by0.08%

Worsened metrics

Total package byte prevSize

241639

decreased by-0.02%

No dependency changes