@overturebio-stack/lectern-client - npm Package Compare versions

Comparing version 1.4.0 to 1.5.0

lib/schema-entities.d.ts

@@ -17,2 +17,3 @@ import { DeepReadonly } from 'deep-freeze';
     readonly description: string;
+    readonly restrictions: SchemaRestriction;
     readonly fields: ReadonlyArray<FieldDefinition>;
@@ -28,2 +29,3 @@ }
 }
+export declare type SchemaData = ReadonlyArray<DataRecord>;
 export declare type FieldChanges = {
@@ -41,2 +43,12 @@ [field: string]: FieldChanges;
 }
+export interface SchemaRestriction {
+    foreignKey?: {
+        schema: string;
+        mappings: {
+            local: string;
+            foreign: string;
+        }[];
+    }[];
+    uniqueKey?: string[];
+}
 export interface FieldDefinition {
@@ -57,2 +69,3 @@ name: string;
         required?: boolean;
+        unique?: boolean;
         range?: RangeRestriction;
@@ -90,3 +103,6 @@ };
     INVALID_ENUM_VALUE = "INVALID_ENUM_VALUE",
-    UNRECOGNIZED_FIELD = "UNRECOGNIZED_FIELD"
+    UNRECOGNIZED_FIELD = "UNRECOGNIZED_FIELD",
+    INVALID_BY_UNIQUE = "INVALID_BY_UNIQUE",
+    INVALID_BY_FOREIGN_KEY = "INVALID_BY_FOREIGN_KEY",
+    INVALID_BY_UNIQUE_KEY = "INVALID_BY_UNIQUE_KEY"
 }
@@ -93,0 +109,0 @@ export interface SchemaValidationError {

lib/schema-entities.js

@@ -52,3 +52,6 @@ "use strict";
     SchemaValidationErrorTypes["UNRECOGNIZED_FIELD"] = "UNRECOGNIZED_FIELD";
+    SchemaValidationErrorTypes["INVALID_BY_UNIQUE"] = "INVALID_BY_UNIQUE";
+    SchemaValidationErrorTypes["INVALID_BY_FOREIGN_KEY"] = "INVALID_BY_FOREIGN_KEY";
+    SchemaValidationErrorTypes["INVALID_BY_UNIQUE_KEY"] = "INVALID_BY_UNIQUE_KEY";
 })(SchemaValidationErrorTypes = exports.SchemaValidationErrorTypes || (exports.SchemaValidationErrorTypes = {}));
 //# sourceMappingURL=schema-entities.js.map

lib/schema-error-messages.js

@@ -21,2 +21,33 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const lodash_1 = require("lodash");
+function getForeignKeyErrorMsg(errorData) {
+    const valueEntries = Object.entries(errorData.info.value);
+    const formattedKeyValues = valueEntries.map(([key, value]) => {
+        if (lodash_1.isArray(value)) {
+            return `${key}: [${value.join(', ')}]`;
+        }
+        else {
+            return `${key}: ${value}`;
+        }
+    });
+    const valuesAsString = formattedKeyValues.join(', ');
+    const detail = `Key ${valuesAsString} is not present in schema ${errorData.info.foreignSchema}`;
+    const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`;
+    return msg;
+}
+function getUniqueKeyErrorMsg(errorData) {
+    const uniqueKeyFields = errorData.info.uniqueKeyFields;
+    const formattedKeyValues = uniqueKeyFields.map(fieldName => {
+        const value = errorData.info.value[fieldName];
+        if (lodash_1.isArray(value)) {
+            return `${fieldName}: [${value.join(', ')}]`;
+        }
+        else {
+            return `${fieldName}: ${value === '' ? 'null' : value}`;
+        }
+    });
+    const valuesAsString = formattedKeyValues.join(', ');
+    const msg = `Key ${valuesAsString} must be unique.`;
+    return msg;
+}
 const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.';
@@ -30,2 +61,5 @@ const ERROR_MESSAGES = {
     MISSING_REQUIRED_FIELD: errorData => `${errorData.fieldName} is a required field.`,
+    INVALID_BY_UNIQUE: errorData => `Value for ${errorData.fieldName} must be unique.`,
+    INVALID_BY_FOREIGN_KEY: errorData => getForeignKeyErrorMsg(errorData),
+    INVALID_BY_UNIQUE_KEY: errorData => getUniqueKeyErrorMsg(errorData),
 };
@@ -32,0 +66,0 @@ // Returns the formatted message for the given error key, taking any required properties from the info object

lib/schema-functions.d.ts

@@ -1,6 +0,7 @@
-import { SchemaProcessingResult, FieldNamesByPriorityMap, BatchProcessingResult } from './schema-entities';
+import { SchemaProcessingResult, FieldNamesByPriorityMap, BatchProcessingResult, SchemaData } from './schema-entities';
 import { SchemasDictionary, SchemaDefinition, DataRecord } from './schema-entities';
 export declare const getSchemaFieldNamesWithPriority: (schema: SchemasDictionary, definition: string) => FieldNamesByPriorityMap;
+export declare const processSchemas: (dictionary: SchemasDictionary, schemasData: Record<string, SchemaData>) => Record<string, BatchProcessingResult>;
 export declare const processRecords: (dataSchema: SchemasDictionary, definition: string, records: ReadonlyArray<DataRecord>) => BatchProcessingResult;
 export declare const process: (dataSchema: SchemasDictionary, definition: string, rec: Readonly<DataRecord>, index: number) => SchemaProcessingResult;
 export declare type ProcessingFunction = (schema: SchemaDefinition, rec: Readonly<DataRecord>, index: number) => any;

lib/schema-functions.js

@@ -24,3 +24,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.process = exports.processRecords = exports.getSchemaFieldNamesWithPriority = void 0;
+exports.process = exports.processRecords = exports.processSchemas = exports.getSchemaFieldNamesWithPriority = void 0;
 const vm_1 = __importDefault(require("vm"));
@@ -32,2 +32,3 @@ const schema_entities_1 = require("./schema-entities");
 const lodash_1 = __importDefault(require("lodash"));
+const records_operations_1 = require("./records-operations");
 const L = logger_1.loggerFor(__filename);
@@ -50,2 +51,49 @@ exports.getSchemaFieldNamesWithPriority = (schema, definition) => {
 };
+const getNotNullSchemaDefinitionFromDictionary = (dictionary, schemaName) => {
+    const schemaDef = dictionary.schemas.find(e => e.name === schemaName);
+    if (!schemaDef) {
+        throw new Error(`no schema found for : ${schemaName}`);
+    }
+    return schemaDef;
+};
+exports.processSchemas = (dictionary, schemasData) => {
+    utils_1.Checks.checkNotNull('dictionary', dictionary);
+    utils_1.Checks.checkNotNull('schemasData', schemasData);
+    const results = {};
+    Object.keys(schemasData).forEach((schemaName) => {
+        // Run validations at the record level
+        const recordLevelValidationResults = exports.processRecords(dictionary, schemaName, schemasData[schemaName]);
+        // Run cross-schema validations
+        const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName);
+        const crossSchemaLevelValidationResults = validation
+            .runCrossSchemaValidationPipeline(schemaDef, schemasData, [
+            validation.validateForeignKey,
+        ])
+            .filter(utils_1.notEmpty);
+        const recordLevelErrors = recordLevelValidationResults.validationErrors.map(x => {
+            return {
+                errorType: x.errorType,
+                index: x.index,
+                fieldName: x.fieldName,
+                info: x.info,
+                message: x.message
+            };
+        });
+        const crossSchemaLevelErrors = crossSchemaLevelValidationResults.map(x => {
+            return {
+                errorType: x.errorType,
+                index: x.index,
+                fieldName: x.fieldName,
+                info: x.info,
+                message: x.message
+            };
+        });
+        const allErrorsBySchema = [...recordLevelErrors, ...crossSchemaLevelErrors];
+        results[schemaName] = utils_1.F({
+            validationErrors: allErrorsBySchema,
+            processedRecords: recordLevelValidationResults.processedRecords
+        });
+    });
+    return results;
+};
 exports.processRecords = (dataSchema, definition, records) => {
@@ -55,6 +103,3 @@ utils_1.Checks.checkNotNull('records', records);
     utils_1.Checks.checkNotNull('definition', definition);
-    const schemaDef = dataSchema.schemas.find(e => e.name === definition);
-    if (!schemaDef) {
-        throw new Error(`no schema found for : ${definition}`);
-    }
+    const schemaDef = getNotNullSchemaDefinitionFromDictionary(dataSchema, definition);
     let validationErrors = [];
@@ -67,2 +112,5 @@ const processedRecords = [];
     });
+    // Record set level validations
+    const newErrors = validateRecordsSet(schemaDef, processedRecords);
+    validationErrors.push(...newErrors);
     L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`);
@@ -199,2 +247,20 @@ return utils_1.F({
 /**
+ * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset.
+ * @param dataset Dataset to select fields from.
+ * @param fields Array with names of the fields to select.
+ * @returns A tuple array. In each tuple, the first element is the index of the row in the dataset, and the second value is the record with the
+ * selected values.
+ */
+const selectFieldsFromDataset = (dataset, fields) => {
+    const records = [];
+    dataset.forEach((row, index) => {
+        const values = {};
+        fields.forEach(field => {
+            values[field] = row[field] || '';
+        });
+        records.push([index, values]);
+    });
+    return records;
+};
+/**
  * Run schema validation pipeline for a schema defintion on the list of records provided.
@@ -242,2 +308,18 @@ * @param definition the schema definition name.
     };
+    validation.runDatasetValidationPipeline = (dataset, schemaDef, funs) => {
+        let result = [];
+        for (const fun of funs) {
+            const typedFunc = fun;
+            result = result.concat(typedFunc(dataset, schemaDef));
+        }
+        return result;
+    };
+    validation.runCrossSchemaValidationPipeline = (schemaDef, schemasData, funs) => {
+        let result = [];
+        for (const fun of funs) {
+            const typedFunc = fun;
+            result = result.concat(typedFunc(schemaDef, schemasData));
+        }
+        return result;
+    };
     validation.validateRegex = (rec, index, fields) => {
@@ -256,5 +338,3 @@ return fields
                 const examples = (_b = field.meta) === null || _b === void 0 ? void 0 : _b.examples;
-                const info = field.isArray
-                    ? { value: invalidValues, regex, examples }
-                    : { regex, examples };
+                const info = { value: invalidValues, regex, examples };
                 return buildError(schema_entities_1.SchemaValidationErrorTypes.INVALID_BY_REGEX, field.name, index, info);
@@ -311,3 +391,3 @@ }
             if (invalidValues.length !== 0) {
-                const info = field.isArray ? { value: invalidValues } : undefined;
+                const info = { value: invalidValues };
                 return buildError(schema_entities_1.SchemaValidationErrorTypes.INVALID_ENUM_VALUE, field.name, index, info);
@@ -319,2 +399,34 @@ }
     };
+    validation.validateUnique = (dataset, schemaDef) => {
+        const errors = [];
+        schemaDef.fields
+            .forEach(field => {
+            var _a;
+            const unique = ((_a = field.restrictions) === null || _a === void 0 ? void 0 : _a.unique) || undefined;
+            if (!unique)
+                return undefined;
+            const keysToValidate = selectFieldsFromDataset(dataset, [field.name]);
+            const duplicateKeys = records_operations_1.findDuplicateKeys(keysToValidate);
+            duplicateKeys.forEach(([index, record]) => {
+                const info = { value: record[field.name] };
+                errors.push(buildError(schema_entities_1.SchemaValidationErrorTypes.INVALID_BY_UNIQUE, field.name, index, info));
+            });
+        });
+        return errors;
+    };
+    validation.validateUniqueKey = (dataset, schemaDef) => {
+        var _a;
+        const errors = [];
+        const uniqueKeyRestriction = (_a = schemaDef === null || schemaDef === void 0 ? void 0 : schemaDef.restrictions) === null || _a === void 0 ? void 0 : _a.uniqueKey;
+        if (uniqueKeyRestriction) {
+            const uniqueKeyFields = uniqueKeyRestriction;
+            const keysToValidate = selectFieldsFromDataset(dataset, uniqueKeyFields);
+            const duplicateKeys = records_operations_1.findDuplicateKeys(keysToValidate);
+            duplicateKeys.forEach(([index, record]) => {
+                const info = { value: record, uniqueKeyFields: uniqueKeyFields };
+                errors.push(buildError(schema_entities_1.SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, uniqueKeyFields.join(', '), index, info));
+            });
+        }
+        return errors;
+    };
     validation.validateValueTypes = (rec, index, fields) => {
@@ -366,2 +478,34 @@ return fields
     };
+    validation.validateForeignKey = (schemaDef, schemasData) => {
+        var _a;
+        const errors = [];
+        const foreignKeyDefinitions = (_a = schemaDef === null || schemaDef === void 0 ? void 0 : schemaDef.restrictions) === null || _a === void 0 ? void 0 : _a.foreignKey;
+        if (foreignKeyDefinitions) {
+            foreignKeyDefinitions.forEach(foreignKeyDefinition => {
+                const localSchemaData = schemasData[schemaDef.name] || [];
+                const foreignSchemaData = schemasData[foreignKeyDefinition.schema] || [];
+                // A foreign key can have more than one field, in which case is a composite foreign key.
+                const localFields = foreignKeyDefinition.mappings.map(x => x.local);
+                const foreignFields = foreignKeyDefinition.mappings.map(x => x.foreign);
+                const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local]));
+                // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite.
+                const localValues = selectFieldsFromDataset(localSchemaData, localFields);
+                const foreignValues = selectFieldsFromDataset(foreignSchemaData, foreignFields);
+                // This artificial record in foreignValues allows null references in localValues to be valid.
+                const emptyRow = {};
+                foreignFields.forEach(field => emptyRow[field] = '');
+                foreignValues.push([-1, emptyRow]);
+                const missingForeignKeys = records_operations_1.findMissingForeignKeys(localValues, foreignValues, fieldsMappings);
+                missingForeignKeys.forEach(record => {
+                    const index = record[0];
+                    const info = {
+                        value: record[1],
+                        foreignSchema: foreignKeyDefinition.schema
+                    };
+                    errors.push(buildError(schema_entities_1.SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, localFields.join(', '), index, info));
+                });
+            });
+        }
+        return errors;
+    };
     validation.getValidFields = (errs, fields) => {
@@ -469,2 +613,11 @@ return fields.filter(field => {
 })(validation || (validation = {}));
+function validateRecordsSet(schemaDef, processedRecords) {
+    const validationErrors = validation
+        .runDatasetValidationPipeline(processedRecords, schemaDef, [
+        validation.validateUnique,
+        validation.validateUniqueKey
+    ])
+        .filter(utils_1.notEmpty);
+    return validationErrors;
+}
 //# sourceMappingURL=schema-functions.js.map

package.json

 {
   "name": "@overturebio-stack/lectern-client",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "files": [
@@ -5,0 +5,0 @@ "lib/**/*"

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

166536

increased by17.1%

Number of package files

36

increased by9.09%

Lines of code

1859

increased by20.09%

No dependency changes