@evidence-dev/db-commons - npm Package Compare versions

Comparing version 0.0.0-6b6bcb47 to 0.0.0-6ba0891f

index.d.ts

+export * from './index.cjs';
+
+export type RunQuery<T extends Record<string, unknown>> = (
+	queryString: string,
+	database: T,
+	batchSize?: number
+) => Promise<QueryResult>;
+
+export type EvidenceColumnType = number | boolean | string | Date;
+
+export type GetRunner<T extends Record<string, unknown>> = (
+	opts: T,
+	directory: string
+) => (queryContent: string, queryPath: string, batchSize: number) => Promise<QueryResult>;
+
+export type ConnectionTester<T extends Record<string, unknown>> = (
+	opts: T,
+	directory: string
+) => Promise<boolean>;
+
+/**
+ * @param {boolean} [disableInterpolation = false] Disables the automatic injection of Evidence Source Variables
+ */
+type FileContent = (disableInterpolation?: boolean) => Promise<string>;
+export interface SourceDirectory {
+	[filename: string]: SourceDirectory | FileContent;
+}
+
+export type ProcessSource<T extends Record<string, unknown>> = (
+	opts: T,
+	files: SourceDirectory,
+	utils: {
+		isCached: (name: string, content: string) => boolean;
+		isFiltered: (name: string) => boolean;
+		shouldRun: (name: string, content: string) => boolean;
+		addToCache: (name: string, content: string) => void;
+	}
+) => AsyncIterable<QueryResult & { name: string; content: string }>;

jest.config.mjs

+/*
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+export default {
+	// All imported modules in your tests should be mocked automatically
+	// automock: false,
+
+	// Stop running tests after `n` failures
+	// bail: 0,
+
+	// The directory where Jest should store its cached dependency information
+	// cacheDirectory: "/tmp/jest_rs",
+
+	// Automatically clear mock calls, instances, contexts and results before every test
+	clearMocks: true,
+
+	// Indicates whether the coverage information should be collected while executing the test
+	// collectCoverage: false,
+
+	// An array of glob patterns indicating a set of files for which coverage information should be collected
+	// collectCoverageFrom: undefined,
+
+	// The directory where Jest should output its coverage files
+	// coverageDirectory: undefined,
+
+	// An array of regexp pattern strings used to skip coverage collection
+	// coveragePathIgnorePatterns: [
+	//   "/node_modules/"
+	// ],
+
+	// Indicates which provider should be used to instrument code for coverage
+	coverageProvider: 'v8',
+
+	// A list of reporter names that Jest uses when writing coverage reports
+	// coverageReporters: [
+	//   "json",
+	//   "text",
+	//   "lcov",
+	//   "clover"
+	// ],
+
+	// An object that configures minimum threshold enforcement for coverage results
+	// coverageThreshold: undefined,
+
+	// A path to a custom dependency extractor
+	// dependencyExtractor: undefined,
+
+	// Make calling deprecated APIs throw helpful error messages
+	// errorOnDeprecated: false,
+
+	// The default configuration for fake timers
+	// fakeTimers: {
+	//   "enableGlobally": false
+	// },
+
+	// Force coverage collection from ignored files using an array of glob patterns
+	// forceCoverageMatch: [],
+
+	// A path to a module which exports an async function that is triggered once before all test suites
+	// globalSetup: undefined,
+
+	// A path to a module which exports an async function that is triggered once after all test suites
+	// globalTeardown: undefined,
+
+	// A set of global variables that need to be available in all test environments
+	// globals: {},
+
+	// The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+	// maxWorkers: "50%",
+
+	// An array of directory names to be searched recursively up from the requiring module's location
+	// moduleDirectories: [
+	//   "node_modules"
+	// ],
+
+	// An array of file extensions your modules use
+	// moduleFileExtensions: [
+	//   "js",
+	//   "mjs",
+	//   "cjs",
+	//   "jsx",
+	//   "ts",
+	//   "tsx",
+	//   "json",
+	//   "node"
+	// ],
+
+	// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+	// moduleNameMapper: {},
+
+	// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+	// modulePathIgnorePatterns: [],
+
+	// Activates notifications for test results
+	// notify: false,
+
+	// An enum that specifies notification mode. Requires { notify: true }
+	// notifyMode: "failure-change",
+
+	// A preset that is used as a base for Jest's configuration
+	// preset: undefined,
+
+	// Run tests from one or more projects
+	// projects: undefined,
+
+	// Use this configuration option to add custom reporters to Jest
+	// reporters: undefined,
+
+	// Automatically reset mock state before every test
+	// resetMocks: false,
+
+	// Reset the module registry before running each individual test
+	// resetModules: false,
+
+	// A path to a custom resolver
+	// resolver: undefined,
+
+	// Automatically restore mock state and implementation before every test
+	// restoreMocks: false,
+
+	// The root directory that Jest should scan for tests and modules within
+	// rootDir: undefined,
+
+	// A list of paths to directories that Jest should use to search for files in
+	// roots: [
+	//   "<rootDir>"
+	// ],
+
+	// Allows you to use a custom runner instead of Jest's default test runner
+	// runner: "jest-runner",
+
+	// The paths to modules that run some code to configure or set up the testing environment before each test
+	// setupFiles: [],
+
+	// A list of paths to modules that run some code to configure or set up the testing framework before each test
+	// setupFilesAfterEnv: [],
+
+	// The number of seconds after which a test is considered as slow and reported as such in the results.
+	// slowTestThreshold: 5,
+
+	// A list of paths to snapshot serializer modules Jest should use for snapshot testing
+	// snapshotSerializers: [],
+
+	// The test environment that will be used for testing
+	// testEnvironment: "jest-environment-node",
+
+	// Options that will be passed to the testEnvironment
+	// testEnvironmentOptions: {},
+
+	// Adds a location field to test results
+	// testLocationInResults: false,
+
+	// The glob patterns Jest uses to detect test files
+	testMatch: ['**/?(*.)+(spec|test).cjs']
+
+	// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+	// testPathIgnorePatterns: [
+	//   "/node_modules/"
+	// ],
+
+	// The regexp pattern or array of patterns that Jest uses to detect test files
+	// testRegex: [],
+
+	// This option allows the use of a custom results processor
+	// testResultsProcessor: undefined,
+
+	// This option allows use of a custom test runner
+	// testRunner: "jest-circus/runner",
+
+	// A map from regular expressions to paths to transformers
+	// transform: undefined,
+
+	// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+	// transformIgnorePatterns: [
+	//   "/node_modules/",
+	//   "\\.pnp\\.[^\\/]+$"
+	// ],
+
+	// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+	// unmockedModulePathPatterns: undefined,
+
+	// Indicates whether each individual test should be reported during the run
+	// verbose: undefined,
+
+	// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+	// watchPathIgnorePatterns: [],
+
+	// Whether to use watchman for file crawling
+	// watchman: true,
+};

src/getEnv.cjs

+/**
+ * @param {string[]} keyPath
+ * @param {any} envMap
+ */
+const getEnv = (envMap, ...keyPath) => {
+	const keyPathParts = [...keyPath];
+
+	let location = envMap;
+	while (keyPathParts.length) {
+		location = location[/** @type {string} */ (keyPathParts.shift())];
+		if (Array.isArray(location) && keyPathParts.length) {
+			// We're too soon
+			throw new Error(`Could not find ${keyPath} in env map!`);
+		}
+	}
+	if (Array.isArray(location)) {
+		// We have run through the entire key and landed on an array
+		for (const { key, deprecated } of location) {
+			if (process.env[key]) {
+				if (deprecated)
+					console.warn(
+						`Found environment variable ${key}. It is being used but is currently deprecated.`
+					);
+				return process.env[key];
+			}
+		}
+	} else {
+		throw new Error(`Could not find ${keyPath} in env map!`);
+	}
+};
+
+exports.getEnv = getEnv;

src/getEnv.spec.cjs

+const { getEnv } = require('./getEnv.cjs');
+
+describe('getEnv', () => {
+	it('should be defined', () => {
+		expect(getEnv).toBeDefined();
+	});
+
+	it('should find a 1st level key', () => {
+		process.env['my-key'] = 'some-value';
+		const map = {
+			someArbitraryString: [{ key: 'my-key', deprecated: false }]
+		};
+
+		const output = getEnv(map, 'someArbitraryString');
+
+		expect(output).toEqual('some-value');
+
+		delete process.env['my-key'];
+	});
+
+	it('should find a 2nd level key', () => {
+		process.env['my-key'] = 'some-value';
+		const map = {
+			'1st': {
+				'2nd': [{ key: 'my-key', deprecated: false }]
+			}
+		};
+		const output = getEnv(map, '1st', '2nd');
+
+		expect(output).toEqual('some-value');
+
+		delete process.env['my-key'];
+	});
+
+	it('should always return the first match', () => {
+		process.env['my-key'] = 'some-value';
+		process.env['my-other-key'] = 'some-other-value';
+		const map = {
+			someArbitraryString: [
+				{ key: 'my-key', deprecated: false },
+				{ key: 'my-other-key', deprecated: false }
+			]
+		};
+
+		const output = getEnv(map, 'someArbitraryString');
+
+		expect(output).toEqual('some-value');
+
+		delete process.env['my-key'];
+		delete process.env['my-other-key'];
+	});
+
+	it('should skip missing values', () => {
+		process.env['my-other-key'] = 'some-other-value';
+		const map = {
+			someArbitraryString: [
+				{ key: 'my-key', deprecated: false },
+				{ key: 'my-other-key', deprecated: false }
+			]
+		};
+
+		const output = getEnv(map, 'someArbitraryString');
+
+		expect(output).toEqual('some-other-value');
+
+		delete process.env['my-other-key'];
+	});
+});

CHANGELOG.md

 # @evidence-dev/db-commons
 
+## 1.0.4
+
+### Patch Changes
+
+- 1da26c4e: add more leniency for comments in queries
+
+## 1.0.3
+
+### Patch Changes
+
+- 31381835: Add support for EVIDENCE_VAR\_\_ interpolation in source queries
+
+## 1.0.2
+
+### Patch Changes
+
+- fc7fe470: Add support for closeConnection callback when async generator has completed
+
+## 1.0.1
+
+### Patch Changes
+
+- Fix incorrectly published version
+
+## 1.0.0
+
+### Patch Changes
+
+- bf4a112a: Update package.json to use new datasource field
+- cd57ba69: Add new interface for datasources for fine-grained control of output
+- c4822852: Support for streaming results
+- 781d2677: exhaust testconnection streams, improve type inference, add trino/databricks adapters
+- 20127231: Bump all versions so version pinning works
+- 29c149d6: added stricter types to db adapters
+
+## 0.2.1-usql.5
+
+### Patch Changes
+
+- 781d2677: exhaust testconnection streams, improve type inference, add trino/databricks adapters
+
+## 0.2.1-usql.4
+
+### Patch Changes
+
+- Update package.json to use new datasource field
+
+## 0.2.1-usql.3
+
+### Patch Changes
+
+- cd57ba69: Add new interface for datasources for fine-grained control of output
+
+## 0.2.1-usql.2
+
+### Patch Changes
+
+- Support for streaming results
+
+## 0.2.1-usql.1
+
+### Patch Changes
+
+- 20127231: Bump all versions so version pinning works
+
+## 0.2.1-usql.0
+
+### Patch Changes
+
+- 29c149d6: added stricter types to db adapters
+
+## 0.2.0
+
+### Minor Changes
+
+- 4c04edd0: Changed expected environment variables to reduce ambiguity
+
 ## 0.1.3
@@ -4,0 +81,0 @@

index.cjs

@@ -1,15 +0,36 @@
-var EvidenceType;
-(function (EvidenceType) {
-	EvidenceType['BOOLEAN'] = 'boolean';
-	EvidenceType['NUMBER'] = 'number';
-	EvidenceType['STRING'] = 'string';
-	EvidenceType['DATE'] = 'date';
-})(EvidenceType || (EvidenceType = {}));
+/**
+ * Enum for evidence types
+ * @readonly
+ * @enum {'boolean' | 'number' | 'string' | 'date'}
+ */
+const EvidenceType = /** @type {const} */ ({
+	BOOLEAN: 'boolean',
+	NUMBER: 'number',
+	STRING: 'string',
+	DATE: 'date',
+	BIGINT: 'bigint'
+});
 
-var TypeFidelity;
-(function (TypeFidelity) {
-	TypeFidelity['INFERRED'] = 'inferred';
-	TypeFidelity['PRECISE'] = 'precise';
-})(TypeFidelity || (TypeFidelity = {}));
+/**
+ * Enum for evidence type fidelity
+ * @readonly
+ * @enum {'inferred' | 'precise'}
+ */
+const TypeFidelity = /** @type {const} */ ({
+	INFERRED: 'inferred',
+	PRECISE: 'precise'
+});
 
+/**
+ * @typedef {Object} ColumnDefinition
+ * @property {string} name
+ * @property {EvidenceType} evidenceType
+ * @property {TypeFidelity} typeFidelity
+ */
+
+/**
+ * Infers the evidence type of a column value
+ * @param {unknown} columnValue
+ * @returns {EvidenceType}
+ */
 const inferValueType = function (columnValue) {
@@ -21,2 +42,3 @@ if (typeof columnValue === 'number') {
 	} else if (typeof columnValue === 'string') {
+		/** @type {EvidenceType} */
 		let result = EvidenceType.STRING;
@@ -48,44 +70,138 @@ if (columnValue && (columnValue.match(/-/g) || []).length === 2) {
 
+/**
+ * Infers the evidence type of each column in a set of rows
+ * @param {Record<string, unknown>[]} rows
+ * @returns {ColumnDefinition[] | undefined}
+ */
 const inferColumnTypes = function (rows) {
-	if (rows && rows.length > 0) {
-		let columns = Object.keys(rows[0]);
-		let columnTypes = columns?.map((column) => {
-			let firstRowWithColumnValue = rows.find((element) =>
-				element[column] == null ? false : true
-			);
-			if (firstRowWithColumnValue) {
-				let inferredType = inferValueType(firstRowWithColumnValue[column]);
-				return { name: column, evidenceType: inferredType, typeFidelity: TypeFidelity.INFERRED };
-			} else {
-				return {
-					name: column,
-					evidenceType: EvidenceType.STRING,
-					typeFidelity: TypeFidelity.INFERRED
-				};
-			}
-		});
-		return columnTypes;
-	}
-	return undefined;
+	if (!rows) return undefined;
+	if (rows.length === 0) return [];
+
+	const columns = Object.keys(rows[0]);
+	const columnTypes = columns.map((column) => {
+		const firstRowWithColumnValue = rows.find((element) => element[column] != null);
+		const inferredType = firstRowWithColumnValue
+			? inferValueType(firstRowWithColumnValue[column])
+			: EvidenceType.STRING;
+		return { name: column, evidenceType: inferredType, typeFidelity: TypeFidelity.INFERRED };
+	});
+
+	return columnTypes;
 };
 
+/**
+ * Processes query results
+ * @param {QueryResult | QueryResult["rows"]} queryResults
+ * @returns {QueryResult}
+ */
 const processQueryResults = function (queryResults) {
-	let rows;
-	let columnTypes;
+	const rows = queryResults.rows ?? queryResults;
+	const columnTypes = queryResults.columnTypes ?? inferColumnTypes(rows);
 
-	if (queryResults.rows) {
-		rows = queryResults.rows;
-	} else {
-		rows = queryResults;
-	}
+	return { rows, columnTypes };
+};
 
-	if (queryResults.columnTypes) {
-		columnTypes = queryResults.columnTypes;
-	} else {
-		columnTypes = inferColumnTypes(rows);
+/**
+ * @typedef {Object} AsyncIterableToBatchedAsyncGeneratorOptions
+ * @property {(rows: Record<string, unknown>[]) => QueryResult["columnTypes"]} [mapResultsToEvidenceColumnTypes]
+ * @property {(row: unknown) => Record<string, unknown>} [standardizeRow]
+ * @property {() => void | Promise<void>} [closeConnection]
+ */
+
+/**
+ * Converts an async iterable to a QueryResult
+ * @param {AsyncIterable<unknown>} iterable
+ * @param {number} batchSize
+ * @param {AsyncIterableToBatchedAsyncGeneratorOptions} options additional optional parameters
+ * @returns {Promise<QueryResult>}
+ */
+const asyncIterableToBatchedAsyncGenerator = async function (
+	iterable,
+	batchSize,
+	{
+		// @ts-ignore
+		standardizeRow = (x) => x,
+		mapResultsToEvidenceColumnTypes,
+		closeConnection = () => {}
+	} = {}
+) {
+	/** @type {Record<string, unknown>[]} */
+	const preread_rows = [];
+
+	/** @type {QueryResult["columnTypes"]} */
+	let columnTypes = [];
+	if (mapResultsToEvidenceColumnTypes) {
+		const iterator = iterable[Symbol.asyncIterator]();
+		const firstRow = await iterator.next().then((x) => x.value);
+		const column_names = Object.keys(firstRow);
+		preread_rows.push(standardizeRow(firstRow));
+
+		let null_columns = column_names.filter((column) => firstRow[column] == null);
+		while (null_columns.length > 0) {
+			const next = await iterator.next().then((x) => x.value);
+			preread_rows.push(standardizeRow(next));
+			null_columns = null_columns.filter((column) => next[column] == null);
+		}
+		columnTypes = mapResultsToEvidenceColumnTypes(preread_rows);
 	}
 
+	const rows = async function* () {
+		let batch = [];
+		batch.push(...preread_rows);
+		for await (const row of iterable) {
+			batch.push(standardizeRow(row));
+			if (batch.length >= batchSize) {
+				yield batch;
+				batch = [];
+			}
+		}
+		// No more batches, safe to close connection now
+		await closeConnection();
+		if (batch.length > 0) {
+			yield batch;
+		}
+		// Clean up
+	};
+
 	return { rows, columnTypes };
 };
 
+/**
+ * Converts an async generator to an array
+ * @param {() => AsyncIterable<Array<Record<string, unknown>>>} asyncGenerator
+ * @returns {Promise<Record<string, unknown>[]>}
+ */
+const batchedAsyncGeneratorToArray = async (asyncGenerator) => {
+	const result = [];
+	for await (const batch of asyncGenerator()) {
+		result.push(...batch);
+	}
+	return result;
+};
+
+/**
+ *
+ * @param {string} query
+ * @returns {string}
+ */
+const cleanQuery = (query) => {
+	let cleanedString = query.trim();
+	if (cleanedString.endsWith(';'))
+		cleanedString = cleanedString.substring(0, cleanedString.length - 1);
+	// query might end with a line comment, which has to be ended
+	return cleanedString + '\n';
+};
+
+/**
+ * @param {QueryResult} stream
+ * @returns {Promise<void>}
+ */
+const exhaustStream = async ({ rows }) => {
+	try {
+		for await (const _ of rows()) {
+			// exhaust the stream
+		}
+	} catch {}
+};
+
 exports.EvidenceType = EvidenceType;
@@ -95,1 +211,7 @@ exports.TypeFidelity = TypeFidelity;
 exports.inferColumnTypes = inferColumnTypes;
+exports.asyncIterableToBatchedAsyncGenerator = asyncIterableToBatchedAsyncGenerator;
+exports.batchedAsyncGeneratorToArray = batchedAsyncGeneratorToArray;
+exports.cleanQuery = cleanQuery;
+exports.exhaustStream = exhaustStream;
+
+exports.getEnv = require('./src/getEnv.cjs').getEnv;

LICENSE.md

@@ -1,6 +0,4 @@
-# License
-
 MIT License
 
-Copyright \(c\) 2021 Evidence
+Copyright \(c\) 2023 Evidence
 
@@ -7,0 +5,0 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files \(the "Software"\), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

package.json

 {
   "name": "@evidence-dev/db-commons",
-  "version": "0.0.0-6b6bcb47",
-  "description": "Shared modules for Evidence Database Drivers ",
+  "version": "0.0.0-6ba0891f",
+  "description": "Shared modules for Evidence Datasource Drivers ",
   "main": "index.cjs",
@@ -9,5 +9,12 @@ "author": "evidence.dev",
   "dependencies": {
-    "fs-extra": "10.0.0"
+    "fs-extra": "11.2.0"
   },
-  "type": "module"
+  "type": "module",
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "jest": "^28.1.3"
+  },
+  "scripts": {
+    "test": "jest"
+  }
 }

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

19246

349.25%

Number of package files

8

100%

Lines of code

448

433.33%

Worsened metrics

Dev dependency count

2

Infinity%

Number of low supply chain risk alerts

13

1200%

Number of medium supply chain risk alerts

1

Infinity%

Dependency changes

• + Added

  fs-extra@11.2.0
  (transitive)

• - Removed

  fs-extra@10.0.0
  (transitive)

• Updated

  fs-extra@11.2.0