🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP
Sign In

@deepnote/database-integrations

Package Overview
Dependencies
Maintainers
4
Versions
17
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@deepnote/database-integrations - npm Package Compare versions

Comparing version
1.4.3
to
1.5.0
+580
-1
dist/index.cjs

@@ -26,2 +26,6 @@ //#region rolldown:runtime

zod = __toESM(zod);
let yaml = require("yaml");
yaml = __toESM(yaml);
let __deepnote_blocks = require("@deepnote/blocks");
__deepnote_blocks = __toESM(__deepnote_blocks);

@@ -101,2 +105,3 @@ //#region src/sql-integration-auth-methods.ts

});
const cloudSqlMetadataSchema = zod.z.object({ service_account: zod.z.string() });
const databricksMetadataSchema = zod.z.object({

@@ -303,2 +308,3 @@ host: zod.z.string(),

"clickhouse": clickhouseMetadataSchema,
"cloud-sql": cloudSqlMetadataSchema,
"databricks": databricksMetadataSchema,

@@ -348,2 +354,7 @@ "dremio": dremioMetadataSchema,

commonIntegrationConfig.extend({
type: zod.default.literal("cloud-sql"),
metadata: databaseMetadataSchemasByType["cloud-sql"],
federated_auth_method: zod.default.null().optional()
}),
commonIntegrationConfig.extend({
type: zod.default.literal("databricks"),

@@ -574,2 +585,3 @@ metadata: databaseMetadataSchemasByType.databricks,

case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
case "cloud-sql": return null;
case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata);

@@ -869,2 +881,3 @@ case "dremio": return getDremioSqlAlchemyInput(integration.metadata);

"clickhouse",
"cloud-sql",
"databricks",

@@ -908,2 +921,196 @@ "dremio",

//#endregion
//#region src/loading/api-error.ts
/**
* Error thrown when a Deepnote API request fails.
*/
var ApiError = class extends Error {
statusCode;
constructor(statusCode, message) {
super(message);
this.name = "ApiError";
this.statusCode = statusCode;
}
};
//#endregion
//#region src/loading/constants.ts
/** Built-in integrations that don't require external configuration. */
const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]);
/** Default `.env` file name for storing integration secrets. */
const DEFAULT_ENV_FILE = ".env";
/** Default integrations file name. */
const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml";
/** Default Deepnote API base URL. */
const DEFAULT_API_URL = "https://api.deepnote.com";
//#endregion
//#region src/loading/env-var-refs.ts
const ENV_VAR_REF_PREFIX = "env:";
function generateEnvVarName(integrationId, fieldPath) {
return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`;
}
function parseEnvVarRef(value) {
if (typeof value !== "string") return null;
if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null;
const varName = value.slice(4);
if (!varName) return null;
return {
prefix: "env",
varName
};
}
function createEnvVarRef(varName) {
return `${ENV_VAR_REF_PREFIX}${varName}`;
}
function isEnvVarRef(value) {
return typeof value === "string" && parseEnvVarRef(value) !== null;
}
function extractEnvVarName(value) {
if (typeof value !== "string") return null;
return parseEnvVarRef(value)?.varName ?? null;
}
/**
* Error thrown when an environment variable reference cannot be resolved.
*/
var EnvVarResolutionError = class extends Error {
varName;
path;
constructor(varName, path) {
const pathInfo = path ? ` at "${path}"` : "";
super(`Environment variable "${varName}" is not defined${pathInfo}`);
this.name = "EnvVarResolutionError";
this.varName = varName;
this.path = path;
}
};
/**
* Recursively resolves environment variable references in an object using an explicit variable map.
* Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map.
*
* @param obj - The object to process (can be any value: primitive, array, or object)
* @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store)
* @param currentPath - Internal parameter for tracking the current path in the object (for error messages)
* @returns A new object with all env var references resolved
* @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved
*/
function resolveEnvVarRefsFromMap(obj, vars, currentPath) {
if (obj === null || obj === void 0) return obj;
if (typeof obj === "string") {
const parsed = parseEnvVarRef(obj);
if (parsed) {
const envValue = vars[parsed.varName];
if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath);
return envValue;
}
return obj;
}
if (Array.isArray(obj)) return obj.map((item, index) => {
return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`);
});
if (typeof obj === "object") {
const result = {};
for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key);
return result;
}
return obj;
}
//#endregion
//#region src/loading/fetch-integrations.ts
/**
* Schema for a single integration from the API.
*/
const apiIntegrationSchema = zod.z.object({
id: zod.z.string(),
name: zod.z.string(),
type: zod.z.string(),
metadata: zod.z.unknown(),
is_public: zod.z.boolean(),
created_at: zod.z.string(),
updated_at: zod.z.string(),
federated_auth_method: zod.z.string().nullable()
}).passthrough();
/**
* Schema for the full API response.
*/
const apiResponseSchema = zod.z.object({ integrations: zod.z.array(apiIntegrationSchema) }).passthrough();
/**
* Fetch integrations from the Deepnote API.
*
* Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike.
*
* @param baseUrl - The base URL of the Deepnote API
* @param token - The authentication token
* @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned.
* @returns Array of integrations from the API
* @throws ApiError if the request fails
*/
async function fetchIntegrations(baseUrl, token, integrationIds) {
const endpoint = new URL(`${baseUrl}/v2/integrations`);
endpoint.searchParams.set("includeMetadata", "true");
if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(","));
const url = endpoint.toString();
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json"
}
});
if (!response.ok) {
if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token.");
if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations.");
throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`);
}
const json = await response.json();
const parseResult = apiResponseSchema.safeParse(json);
if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`);
return parseResult.data.integrations;
}
//#endregion
//#region src/loading/integrations-document.ts
/**
* Parse integrations YAML content into a mutable `yaml` Document that preserves
* comments and formatting. Returns `null` for empty content.
*
* This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`.
*/
function parseIntegrationsDocument(content) {
if (!content.trim()) return null;
return (0, yaml.parseDocument)(content, {
strict: true,
version: "1.2"
});
}
/**
* Serialize an integrations Document back to YAML text, preserving comments and
* formatting and avoiding line wrapping.
*/
function serializeIntegrationsDocument(doc) {
return doc.toString({ lineWidth: 0 });
}
//#endregion
//#region src/loading/integrations-file-schema.ts
/**
* Schema for the integrations YAML file structure.
* Uses loose validation to accept any array of objects,
* allowing per-entry validation to report specific issues.
*
* Example file format:
* ```yaml
* integrations:
* - id: my-postgres
* name: My PostgreSQL
* type: pgsql
* metadata:
* host: localhost
* ...
* ```
*/
const baseIntegrationsFileSchema = zod.z.object({ integrations: zod.z.array(zod.z.record(zod.z.unknown())).optional().default([]) });
const integrationsFileSchema = zod.z.object({ integrations: zod.z.array(databaseIntegrationConfigSchema) });
//#endregion
//#region src/secret-field-paths.ts

@@ -930,2 +1137,3 @@ /**

spanner: ["service_account"],
"cloud-sql": ["service_account"],
mongodb: [

@@ -967,9 +1175,362 @@ "password",

//#endregion
//#region src/loading/merge-integrations.ts
/**
* JSON schema URL for the integrations file.
* TODO - change to main branch when the PR is merged into main
*/
const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json";
/**
* Schema comment for the YAML file.
*/
const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`;
/**
* Error thrown when the integrations property has an invalid type.
*/
var InvalidIntegrationsTypeError = class extends Error {
constructor() {
super(`Invalid 'integrations' property: expected a list`);
this.name = "InvalidIntegrationsTypeError";
}
};
/**
* Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist.
* If the integrations property is missing or null, creates an empty array and returns a reference to it.
* Throws InvalidIntegrationsTypeError if integrations exists but is not an array.
*/
function getOrCreateIntegrationsFromDocument(doc) {
const integrations = doc.get("integrations");
if (integrations == null) {
const emptySeq = doc.createNode([]);
emptySeq.flow = false;
doc.set("integrations", emptySeq);
return emptySeq;
}
if (!(0, yaml.isSeq)(integrations)) throw new InvalidIntegrationsTypeError();
return integrations;
}
function getOrCreateIntegrationMetadata(doc, integrationMap) {
const existingMetadata = integrationMap.get("metadata", true);
if ((0, yaml.isMap)(existingMetadata)) return existingMetadata;
const metadata = doc.createNode({});
integrationMap.set("metadata", metadata);
return metadata;
}
function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) {
const secrets = {};
for (const [key, value] of Object.entries(integrationMetadata)) {
if (value === void 0) continue;
if (value === null) {
metadataMap.set(key, null);
continue;
}
if (!secretPaths.includes(key)) {
metadataMap.set(key, value);
continue;
}
const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key);
secrets[envVarName] = String(value);
metadataMap.set(key, createEnvVarRef(envVarName));
}
return secrets;
}
/**
* Update an existing integration entry in the document.
* Preserves any comments or extra fields while updating known fields.
* Extracts secrets and replaces them with env var references, preserving custom env var names.
*
* @returns Record of extracted secrets (envVarName -> secretValue)
*/
function updateIntegrationInDocument(doc, integrationMap, integration) {
const { id, name, type, metadata, federated_auth_method,...rest } = integration;
integrationMap.set("id", id);
integrationMap.set("name", name);
integrationMap.set("type", type);
const secretPaths = getSecretFieldPaths(type);
const secrets = updateIntegrationMetadataMap({
metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap),
integrationId: id,
integrationMetadata: metadata,
secretPaths
});
if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method);
else integrationMap.delete("federated_auth_method");
return secrets;
}
/**
* Add a new integration to the sequence.
* Extracts secrets and replaces them with env var references.
*
* @returns Record of extracted secrets (envVarName -> secretValue)
*/
function addIntegrationToSeq(doc, integrationsSeq, integration) {
const secretPaths = getSecretFieldPaths(integration.type);
const { metadata,...restIntegration } = integration;
const integrationMap = doc.createNode(restIntegration);
const metadataMap = doc.createNode({});
const secrets = updateIntegrationMetadataMap({
metadataMap,
integrationId: integration.id,
integrationMetadata: metadata,
secretPaths
});
integrationMap.setIn(["metadata"], metadataMap);
integrationsSeq.add(integrationMap);
return secrets;
}
/**
* Create a new Document with the schema comment.
*/
function createNewDocument() {
const doc = (0, yaml.parseDocument)("integrations: []\n", { version: "1.2" });
doc.commentBefore = SCHEMA_COMMENT;
const integrations = doc.get("integrations", true);
if ((0, yaml.isSeq)(integrations)) integrations.flow = false;
return doc;
}
/**
* Merge processed integrations into the document.
* Updates existing entries in-place (preserving comments) and adds new ones.
* Extracts secrets and replaces them with env var references during the merge.
* Preserves custom environment variable names from existing entries.
*
* @returns Object containing existing count and extracted secrets
*/
function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) {
const existingCount = integrationsSeq.items.length;
let updatedCount = 0;
let newCount = 0;
const secrets = {};
for (const databaseIntegration of processedIntegrations) {
const existingIntegration = integrationsSeq.items.find((item) => {
if (!(0, yaml.isMap)(item)) return false;
return item.get("id") === databaseIntegration.id;
});
if (existingIntegration != null) {
const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration);
Object.assign(secrets, extractedSecrets);
updatedCount++;
} else {
const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration);
Object.assign(secrets, extractedSecrets);
newCount++;
}
}
return {
secrets,
stats: {
existingCount,
newCount,
updatedCount
}
};
}
/**
* Error thrown when an API integration fails schema validation during conversion.
*/
var InvalidIntegrationError = class extends Error {
integrationId;
constructor(integrationId, message) {
super(message);
this.name = "InvalidIntegrationError";
this.integrationId = integrationId;
}
};
/**
* Convert API integrations to DatabaseIntegrationConfig.
* Validates each integration against the schema and collects errors for invalid ones.
*
* Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching).
*
* @param apiIntegrations - Integrations fetched from the API
* @returns Object containing validated integrations and any validation errors
*/
function convertApiIntegrations(apiIntegrations) {
const integrations = [];
const errors = [];
for (const apiIntegration of apiIntegrations) {
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
if (!config.success) {
errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API."));
continue;
}
integrations.push(config.data);
}
return {
integrations,
errors
};
}
/**
* Merge API integrations into an existing document (or create a new one).
* Extracts secrets and replaces them with env var references during the merge.
* Preserves custom environment variable names from existing entries.
* Invalid or unsupported integrations are silently skipped.
*
* @param doc - The YAML document to merge into
* @param apiIntegrations - Integrations fetched from the API
* @returns The extracted secrets and merge statistics
*/
function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) {
const integrationsSeq = getOrCreateIntegrationsFromDocument(doc);
const skipped = [];
return {
...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => {
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
if (!config.success) {
skipped.push({
id: apiIntegration.id,
name: apiIntegration.name,
type: apiIntegration.type,
issues: config.error.issues
});
return acc;
}
acc.push(config.data);
return acc;
}, [])),
skipped
};
}
//#endregion
//#region src/loading/merge-into-yaml.ts
/**
* Merge API integrations into existing integrations YAML content (or create a new
* document), returning the serialized YAML, the extracted secrets, and merge stats.
*
* This is a string-in / string-out convenience that mirrors what `deepnote
* integrations pull` does to the YAML file — usable without touching the `yaml`
* Document API. Callers persist `content` and `secrets` however they like (e.g.
* the CLI writes files; the VS Code extension can write via `workspace.fs`).
*
* @param existingContent - Current YAML content, or `null` if the file doesn't exist yet
* @param apiIntegrations - Integrations fetched from the API
*/
function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) {
const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument();
const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations);
if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT;
return {
content: serializeIntegrationsDocument(doc),
secrets,
stats,
skipped
};
}
//#endregion
//#region src/loading/parse-integrations.ts
/**
* Format Zod validation errors into ValidationIssue format.
*/
function formatZodIssues(issues, pathPrefix) {
return issues.map((issue) => ({
path: [pathPrefix, ...issue.path].filter(Boolean).join("."),
message: issue.message,
code: issue.code
}));
}
/**
* Parse integrations YAML content gracefully.
* Returns all valid integrations and collects validation issues for invalid ones.
*
* This is the content-accepting, environment-agnostic core (no filesystem or
* `process.env` access) — callers supply the YAML text and the variable map used
* to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this
* to read from disk and default the map to `process.env`.
*
* @returns Object containing valid integrations and any validation issues
*/
function parseIntegrations({ yaml: yaml$1, env = {} }) {
const emptyIntegrations = [];
const issues = [];
let parsed;
try {
parsed = (0, __deepnote_blocks.parseYaml)(yaml$1);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
issues.push({
path: "",
message: `Invalid YAML in integrations file: ${message}`,
code: "yaml_parse_error"
});
return {
integrations: emptyIntegrations,
issues
};
}
if (parsed === null || parsed === void 0) return {
integrations: emptyIntegrations,
issues
};
const fileResult = baseIntegrationsFileSchema.safeParse(parsed);
if (!fileResult.success) {
issues.push(...formatZodIssues(fileResult.error.issues, ""));
return {
integrations: emptyIntegrations,
issues
};
}
const entries = fileResult.data.integrations;
const integrations = [];
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
const pathPrefix = `integrations[${i}]`;
const entryId = zod.z.string().safeParse(entry.id).data;
const integrationLabel = zod.z.string().safeParse(entry.name).data || entryId;
let resolvedEntry;
try {
resolvedEntry = resolveEnvVarRefsFromMap(entry, env);
} catch (error) {
if (error instanceof EnvVarResolutionError) {
const context = integrationLabel ? `Integration "${integrationLabel}": ` : "";
issues.push({
path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix,
message: `${context}${error.message}`,
code: "env_var_not_defined"
});
continue;
}
throw error;
}
const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry);
if (result.success) integrations.push(result.data);
else {
const formattedIssues = formatZodIssues(result.error.issues, pathPrefix);
if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`;
issues.push(...formattedIssues);
}
}
return {
integrations,
issues
};
}
//#endregion
exports.ApiError = ApiError;
exports.AwsAuthMethods = AwsAuthMethods;
exports.BUILTIN_INTEGRATIONS = BUILTIN_INTEGRATIONS;
exports.BigQueryAuthMethods = BigQueryAuthMethods;
exports.BigQueryServiceAccountParseError = BigQueryServiceAccountParseError;
exports.DEFAULT_API_URL = DEFAULT_API_URL;
exports.DEFAULT_ENV_FILE = DEFAULT_ENV_FILE;
exports.DEFAULT_INTEGRATIONS_FILE = DEFAULT_INTEGRATIONS_FILE;
exports.DatabaseAuthMethods = DatabaseAuthMethods;
exports.ENV_VAR_REF_PREFIX = ENV_VAR_REF_PREFIX;
exports.EnvVarResolutionError = EnvVarResolutionError;
exports.InvalidIntegrationError = InvalidIntegrationError;
exports.InvalidIntegrationsTypeError = InvalidIntegrationsTypeError;
exports.JSON_SCHEMA_URL = JSON_SCHEMA_URL;
exports.SCHEMA_COMMENT = SCHEMA_COMMENT;
exports.SnowflakeAuthMethods = SnowflakeAuthMethods;
exports.SpannerServiceAccountParseError = SpannerServiceAccountParseError;
exports.TrinoAuthMethods = TrinoAuthMethods;
exports.addIntegrationToSeq = addIntegrationToSeq;
exports.apiIntegrationSchema = apiIntegrationSchema;
exports.apiResponseSchema = apiResponseSchema;
exports.baseIntegrationsFileSchema = baseIntegrationsFileSchema;
exports.convertApiIntegrations = convertApiIntegrations;
exports.createEnvVarRef = createEnvVarRef;
exports.createNewDocument = createNewDocument;
exports.databaseIntegrationConfigSchema = databaseIntegrationConfigSchema;

@@ -979,12 +1540,30 @@ exports.databaseIntegrationTypes = databaseIntegrationTypes;

exports.databaseMetadataSchemasByType = databaseMetadataSchemasByType;
exports.extractEnvVarName = extractEnvVarName;
exports.federatedAuthMethods = federatedAuthMethods;
exports.fetchIntegrations = fetchIntegrations;
exports.generateEnvVarName = generateEnvVarName;
exports.getEnvironmentVariablesForIntegrations = getEnvironmentVariablesForIntegrations;
exports.getOrCreateIntegrationMetadata = getOrCreateIntegrationMetadata;
exports.getOrCreateIntegrationsFromDocument = getOrCreateIntegrationsFromDocument;
exports.getSecretFieldPaths = getSecretFieldPaths;
exports.getSnowflakeFederatedAuthSqlAlchemyInput = getSnowflakeFederatedAuthSqlAlchemyInput;
exports.getSqlAlchemyInput = getSqlAlchemyInput;
exports.getSqlEnvVarName = getSqlEnvVarName;
exports.integrationsFileSchema = integrationsFileSchema;
exports.isDatabaseIntegrationType = isDatabaseIntegrationType;
exports.isDatabaseIntegrationTypeWithSslSupport = isDatabaseIntegrationTypeWithSslSupport;
exports.isEnvVarRef = isEnvVarRef;
exports.isFederatedAuthMetadata = isFederatedAuthMetadata;
exports.isFederatedAuthMethod = isFederatedAuthMethod;
exports.isSqlIntegrationType = isSqlIntegrationType;
exports.sqlIntegrationTypes = sqlIntegrationTypes;
exports.mergeApiIntegrationsIntoDocument = mergeApiIntegrationsIntoDocument;
exports.mergeApiIntegrationsIntoYaml = mergeApiIntegrationsIntoYaml;
exports.mergeProcessedIntegrations = mergeProcessedIntegrations;
exports.parseEnvVarRef = parseEnvVarRef;
exports.parseIntegrations = parseIntegrations;
exports.parseIntegrationsDocument = parseIntegrationsDocument;
exports.resolveEnvVarRefsFromMap = resolveEnvVarRefsFromMap;
exports.serializeIntegrationsDocument = serializeIntegrationsDocument;
exports.sqlIntegrationTypes = sqlIntegrationTypes;
exports.updateIntegrationInDocument = updateIntegrationInDocument;
exports.updateIntegrationMetadataMap = updateIntegrationMetadataMap;
+758
-217

@@ -1,2 +0,4 @@

import z$1, { z } from "zod";
import z, { z as z$1 } from "zod";
import { isMap, isSeq, parseDocument } from "yaml";
import { parseYaml } from "@deepnote/blocks";

@@ -46,86 +48,87 @@ //#region src/sql-integration-auth-methods.ts

//#region src/database-integration-metadata-schemas.ts
const athenaMetadataSchema = z.object({
access_key_id: z.string(),
region: z.string(),
s3_output_path: z.string(),
secret_access_key: z.string(),
workgroup: z.string().optional()
const athenaMetadataSchema = z$1.object({
access_key_id: z$1.string(),
region: z$1.string(),
s3_output_path: z$1.string(),
secret_access_key: z$1.string(),
workgroup: z$1.string().optional()
});
const bigqueryMetadataSchema = z.discriminatedUnion("authMethod", [z.object({
authMethod: z.literal(BigQueryAuthMethods.ServiceAccount).optional(),
service_account: z.string()
}), z.object({
authMethod: z.literal(BigQueryAuthMethods.GoogleOauth),
project: z.string(),
clientId: z.string(),
clientSecret: z.string()
const bigqueryMetadataSchema = z$1.discriminatedUnion("authMethod", [z$1.object({
authMethod: z$1.literal(BigQueryAuthMethods.ServiceAccount).optional(),
service_account: z$1.string()
}), z$1.object({
authMethod: z$1.literal(BigQueryAuthMethods.GoogleOauth),
project: z$1.string(),
clientId: z$1.string(),
clientSecret: z$1.string()
})]);
const clickhouseMetadataSchema = z.object({
host: z.string(),
user: z.string(),
password: z.string().optional(),
database: z.string(),
port: z.string().optional(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
const clickhouseMetadataSchema = z$1.object({
host: z$1.string(),
user: z$1.string(),
password: z$1.string().optional(),
database: z$1.string(),
port: z$1.string().optional(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const databricksMetadataSchema = z.object({
host: z.string(),
httpPath: z.string(),
token: z.string(),
port: z.string(),
schema: z.string().optional(),
catalog: z.string().optional(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional()
const cloudSqlMetadataSchema = z$1.object({ service_account: z$1.string() });
const databricksMetadataSchema = z$1.object({
host: z$1.string(),
httpPath: z$1.string(),
token: z$1.string(),
port: z$1.string(),
schema: z$1.string().optional(),
catalog: z$1.string().optional(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional()
});
const dremioMetadataSchema = z.object({
schema: z.string(),
host: z.string(),
port: z.string(),
token: z.string(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional()
const dremioMetadataSchema = z$1.object({
schema: z$1.string(),
host: z$1.string(),
port: z$1.string(),
token: z$1.string(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional()
});
const mongodbMetadataSchema = z.object({
connection_string: z.string(),
rawConnectionString: z.string().optional(),
prefix: z.string().optional(),
host: z.string().optional(),
port: z.string().optional(),
user: z.string().optional(),
password: z.string().optional(),
database: z.string().optional(),
options: z.string().optional(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
const mongodbMetadataSchema = z$1.object({
connection_string: z$1.string(),
rawConnectionString: z$1.string().optional(),
prefix: z$1.string().optional(),
host: z$1.string().optional(),
port: z$1.string().optional(),
user: z$1.string().optional(),
password: z$1.string().optional(),
database: z$1.string().optional(),
options: z$1.string().optional(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const pandasDataframeMetadataSchema = z.object({});
const commonRedshiftMetadataSchema = z.object({
database: z.string(),
host: z.string(),
port: z.string().optional(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
const pandasDataframeMetadataSchema = z$1.object({});
const commonRedshiftMetadataSchema = z$1.object({
database: z$1.string(),
host: z$1.string(),
port: z$1.string().optional(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const redshiftMetadataSchema = z.preprocess((data) => {
const redshiftMetadataSchema = z$1.preprocess((data) => {
if (typeof data === "object" && data !== null && (!("authMethod" in data) || !data.authMethod)) return {

@@ -136,27 +139,27 @@ ...data,

return data;
}, z.union([
}, z$1.union([
commonRedshiftMetadataSchema.extend({
authMethod: z.literal(DatabaseAuthMethods.UsernameAndPassword).optional(),
user: z.string(),
password: z.string()
authMethod: z$1.literal(DatabaseAuthMethods.UsernameAndPassword).optional(),
user: z$1.string(),
password: z$1.string()
}),
commonRedshiftMetadataSchema.extend({
authMethod: z.literal(AwsAuthMethods.IamRole),
roleArn: z.string(),
roleExternalId: z.string(),
roleNonce: z.string()
authMethod: z$1.literal(AwsAuthMethods.IamRole),
roleArn: z$1.string(),
roleExternalId: z$1.string(),
roleNonce: z$1.string()
}),
commonRedshiftMetadataSchema.extend({ authMethod: z.literal(DatabaseAuthMethods.IndividualCredentials) })
commonRedshiftMetadataSchema.extend({ authMethod: z$1.literal(DatabaseAuthMethods.IndividualCredentials) })
]));
const commonSnowflakeMetadataSchema = z.object({
accountName: z.string(),
warehouse: z.string().optional(),
database: z.string().optional(),
role: z.string().optional(),
dbt: z.boolean().optional(),
dbtServiceToken: z.string().optional(),
dbtPrimaryJobId: z.string().optional(),
dbtProxyServerUrl: z.string().optional()
const commonSnowflakeMetadataSchema = z$1.object({
accountName: z$1.string(),
warehouse: z$1.string().optional(),
database: z$1.string().optional(),
role: z$1.string().optional(),
dbt: z$1.boolean().optional(),
dbtServiceToken: z$1.string().optional(),
dbtPrimaryJobId: z$1.string().optional(),
dbtProxyServerUrl: z$1.string().optional()
});
const snowflakeMetadataSchema = z.preprocess((data) => {
const snowflakeMetadataSchema = z$1.preprocess((data) => {
if (typeof data === "object" && data !== null && (!("authMethod" in data) || !data.authMethod)) return {

@@ -167,109 +170,109 @@ ...data,

return data;
}, z.discriminatedUnion("authMethod", [
}, z$1.discriminatedUnion("authMethod", [
commonSnowflakeMetadataSchema.extend({
authMethod: z.literal(SnowflakeAuthMethods.Password).optional(),
username: z.string(),
password: z.string()
authMethod: z$1.literal(SnowflakeAuthMethods.Password).optional(),
username: z$1.string(),
password: z$1.string()
}),
commonSnowflakeMetadataSchema.extend({
authMethod: z.literal(SnowflakeAuthMethods.Okta),
clientId: z.string(),
clientSecret: z.string(),
oktaSubdomain: z.string(),
identityProvider: z.string(),
authorizationServer: z.string()
authMethod: z$1.literal(SnowflakeAuthMethods.Okta),
clientId: z$1.string(),
clientSecret: z$1.string(),
oktaSubdomain: z$1.string(),
identityProvider: z$1.string(),
authorizationServer: z$1.string()
}),
commonSnowflakeMetadataSchema.extend({
authMethod: z.literal(SnowflakeAuthMethods.NativeSnowflake),
clientId: z.string(),
clientSecret: z.string()
authMethod: z$1.literal(SnowflakeAuthMethods.NativeSnowflake),
clientId: z$1.string(),
clientSecret: z$1.string()
}),
commonSnowflakeMetadataSchema.extend({
authMethod: z.literal(SnowflakeAuthMethods.AzureAd),
clientId: z.string(),
clientSecret: z.string(),
resource: z.string(),
tenant: z.string()
authMethod: z$1.literal(SnowflakeAuthMethods.AzureAd),
clientId: z$1.string(),
clientSecret: z$1.string(),
resource: z$1.string(),
tenant: z$1.string()
}),
commonSnowflakeMetadataSchema.extend({ authMethod: z.literal(SnowflakeAuthMethods.KeyPair) }),
commonSnowflakeMetadataSchema.extend({ authMethod: z$1.literal(SnowflakeAuthMethods.KeyPair) }),
commonSnowflakeMetadataSchema.extend({
authMethod: z.literal(SnowflakeAuthMethods.ServiceAccountKeyPair),
username: z.string(),
privateKey: z.string(),
privateKeyPassphrase: z.string().optional()
authMethod: z$1.literal(SnowflakeAuthMethods.ServiceAccountKeyPair),
username: z$1.string(),
privateKey: z$1.string(),
privateKeyPassphrase: z$1.string().optional()
})
]));
const spannerMetadataSchema = z.object({
service_account: z.string(),
dataBoostEnabled: z.boolean(),
instance: z.string(),
database: z.string()
const spannerMetadataSchema = z$1.object({
service_account: z$1.string(),
dataBoostEnabled: z$1.boolean(),
instance: z$1.string(),
database: z$1.string()
});
const trinoBaseMetadataSchema = z.object({
host: z.string(),
port: z.string().optional(),
database: z.string(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
const trinoBaseMetadataSchema = z$1.object({
host: z$1.string(),
port: z$1.string().optional(),
database: z$1.string(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const trinoUsernamePasswordMetadataSchema = trinoBaseMetadataSchema.extend({
authMethod: z.literal(TrinoAuthMethods.Password).nullish(),
user: z.string(),
password: z.string()
authMethod: z$1.literal(TrinoAuthMethods.Password).nullish(),
user: z$1.string(),
password: z$1.string()
});
const trinoOAuthMetadataSchema = trinoBaseMetadataSchema.extend({
authMethod: z.literal(TrinoAuthMethods.Oauth),
clientId: z.string(),
clientSecret: z.string(),
authUrl: z.string().url(),
tokenUrl: z.string().url()
authMethod: z$1.literal(TrinoAuthMethods.Oauth),
clientId: z$1.string(),
clientSecret: z$1.string(),
authUrl: z$1.string().url(),
tokenUrl: z$1.string().url()
});
const trinoMetadataSchema = z.discriminatedUnion("authMethod", [trinoUsernamePasswordMetadataSchema, trinoOAuthMetadataSchema]);
const commonDatabaseSchema = z.object({
host: z.string(),
user: z.string(),
password: z.string(),
database: z.string(),
sshEnabled: z.boolean().optional(),
sshHost: z.string().optional(),
sshPort: z.string().optional(),
sshUser: z.string().optional()
const trinoMetadataSchema = z$1.discriminatedUnion("authMethod", [trinoUsernamePasswordMetadataSchema, trinoOAuthMetadataSchema]);
const commonDatabaseSchema = z$1.object({
host: z$1.string(),
user: z$1.string(),
password: z$1.string(),
database: z$1.string(),
sshEnabled: z$1.boolean().optional(),
sshHost: z$1.string().optional(),
sshPort: z$1.string().optional(),
sshUser: z$1.string().optional()
});
const alloydbMetadataSchema = commonDatabaseSchema.extend({
port: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
port: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const mariadbMetadataSchema = commonDatabaseSchema.extend({
port: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
port: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const mindsdbMetadataSchema = commonDatabaseSchema.extend({
port: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
port: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const mysqlMetadataSchema = commonDatabaseSchema.extend({
port: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
port: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const pgsqlMetadataSchema = commonDatabaseSchema.extend({
port: z.string().optional(),
sslEnabled: z.boolean().optional(),
caCertificateName: z.string().optional(),
caCertificateText: z.string().optional()
port: z$1.string().optional(),
sslEnabled: z$1.boolean().optional(),
caCertificateName: z$1.string().optional(),
caCertificateText: z$1.string().optional()
});
const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: z.string() });
const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: z.string() });
const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: z$1.string() });
const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: z$1.string() });
const databaseMetadataSchemasByType = {

@@ -280,2 +283,3 @@ "alloydb": alloydbMetadataSchema,

"clickhouse": clickhouseMetadataSchema,
"cloud-sql": cloudSqlMetadataSchema,
"databricks": databricksMetadataSchema,

@@ -299,76 +303,81 @@ "dremio": dremioMetadataSchema,

//#region src/database-integration-config.ts
const commonIntegrationConfig = z$1.object({
id: z$1.string(),
name: z$1.string()
const commonIntegrationConfig = z.object({
id: z.string(),
name: z.string()
});
const databaseIntegrationConfigSchema = z$1.discriminatedUnion("type", [
const databaseIntegrationConfigSchema = z.discriminatedUnion("type", [
commonIntegrationConfig.extend({
type: z$1.literal("alloydb"),
type: z.literal("alloydb"),
metadata: databaseMetadataSchemasByType.alloydb,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("athena"),
type: z.literal("athena"),
metadata: databaseMetadataSchemasByType.athena,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("big-query"),
type: z.literal("big-query"),
metadata: databaseMetadataSchemasByType["big-query"],
federated_auth_method: z$1.enum(["service-account", "google-oauth"]).optional().nullable()
federated_auth_method: z.enum(["service-account", "google-oauth"]).optional().nullable()
}),
commonIntegrationConfig.extend({
type: z$1.literal("clickhouse"),
type: z.literal("clickhouse"),
metadata: databaseMetadataSchemasByType.clickhouse,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("databricks"),
type: z.literal("cloud-sql"),
metadata: databaseMetadataSchemasByType["cloud-sql"],
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z.literal("databricks"),
metadata: databaseMetadataSchemasByType.databricks,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("dremio"),
type: z.literal("dremio"),
metadata: databaseMetadataSchemasByType.dremio,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("mariadb"),
type: z.literal("mariadb"),
metadata: databaseMetadataSchemasByType.mariadb,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("materialize"),
type: z.literal("materialize"),
metadata: databaseMetadataSchemasByType.materialize,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("mindsdb"),
type: z.literal("mindsdb"),
metadata: databaseMetadataSchemasByType.mindsdb,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("mongodb"),
type: z.literal("mongodb"),
metadata: databaseMetadataSchemasByType.mongodb,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("mysql"),
type: z.literal("mysql"),
metadata: databaseMetadataSchemasByType.mysql,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("pandas-dataframe"),
type: z.literal("pandas-dataframe"),
metadata: databaseMetadataSchemasByType["pandas-dataframe"],
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("pgsql"),
type: z.literal("pgsql"),
metadata: databaseMetadataSchemasByType.pgsql,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("redshift"),
type: z.literal("redshift"),
metadata: databaseMetadataSchemasByType.redshift,
federated_auth_method: z$1.enum([
federated_auth_method: z.enum([
"username-and-password",

@@ -380,5 +389,5 @@ "iam-role",

commonIntegrationConfig.extend({
type: z$1.literal("snowflake"),
type: z.literal("snowflake"),
metadata: databaseMetadataSchemasByType.snowflake,
federated_auth_method: z$1.enum([
federated_auth_method: z.enum([
"okta",

@@ -393,15 +402,15 @@ "snowflake",

commonIntegrationConfig.extend({
type: z$1.literal("spanner"),
type: z.literal("spanner"),
metadata: databaseMetadataSchemasByType.spanner,
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("sql-server"),
type: z.literal("sql-server"),
metadata: databaseMetadataSchemasByType["sql-server"],
federated_auth_method: z$1.null().optional()
federated_auth_method: z.null().optional()
}),
commonIntegrationConfig.extend({
type: z$1.literal("trino"),
type: z.literal("trino"),
metadata: databaseMetadataSchemasByType.trino,
federated_auth_method: z$1.enum(["trino-oauth", "password"]).optional().nullable()
federated_auth_method: z.enum(["trino-oauth", "password"]).optional().nullable()
})

@@ -553,2 +562,3 @@ ]);

case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
case "cloud-sql": return null;
case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata);

@@ -848,2 +858,3 @@ case "dremio": return getDremioSqlAlchemyInput(integration.metadata);

"clickhouse",
"cloud-sql",
"databricks",

@@ -887,2 +898,196 @@ "dremio",

//#endregion
//#region src/loading/api-error.ts
/**
* Error thrown when a Deepnote API request fails.
*/
var ApiError = class extends Error {
statusCode;
constructor(statusCode, message) {
super(message);
this.name = "ApiError";
this.statusCode = statusCode;
}
};
//#endregion
//#region src/loading/constants.ts
/** Built-in integrations that don't require external configuration. */
const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]);
/** Default `.env` file name for storing integration secrets. */
const DEFAULT_ENV_FILE = ".env";
/** Default integrations file name. */
const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml";
/** Default Deepnote API base URL. */
const DEFAULT_API_URL = "https://api.deepnote.com";
//#endregion
//#region src/loading/env-var-refs.ts
const ENV_VAR_REF_PREFIX = "env:";
function generateEnvVarName(integrationId, fieldPath) {
return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`;
}
function parseEnvVarRef(value) {
if (typeof value !== "string") return null;
if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null;
const varName = value.slice(4);
if (!varName) return null;
return {
prefix: "env",
varName
};
}
function createEnvVarRef(varName) {
return `${ENV_VAR_REF_PREFIX}${varName}`;
}
function isEnvVarRef(value) {
return typeof value === "string" && parseEnvVarRef(value) !== null;
}
function extractEnvVarName(value) {
if (typeof value !== "string") return null;
return parseEnvVarRef(value)?.varName ?? null;
}
/**
* Error thrown when an environment variable reference cannot be resolved.
*/
var EnvVarResolutionError = class extends Error {
varName;
path;
constructor(varName, path) {
const pathInfo = path ? ` at "${path}"` : "";
super(`Environment variable "${varName}" is not defined${pathInfo}`);
this.name = "EnvVarResolutionError";
this.varName = varName;
this.path = path;
}
};
/**
* Recursively resolves environment variable references in an object using an explicit variable map.
* Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map.
*
* @param obj - The object to process (can be any value: primitive, array, or object)
* @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store)
* @param currentPath - Internal parameter for tracking the current path in the object (for error messages)
* @returns A new object with all env var references resolved
* @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved
*/
function resolveEnvVarRefsFromMap(obj, vars, currentPath) {
if (obj === null || obj === void 0) return obj;
if (typeof obj === "string") {
const parsed = parseEnvVarRef(obj);
if (parsed) {
const envValue = vars[parsed.varName];
if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath);
return envValue;
}
return obj;
}
if (Array.isArray(obj)) return obj.map((item, index) => {
return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`);
});
if (typeof obj === "object") {
const result = {};
for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key);
return result;
}
return obj;
}
//#endregion
//#region src/loading/fetch-integrations.ts
/**
* Schema for a single integration from the API.
*/
const apiIntegrationSchema = z$1.object({
id: z$1.string(),
name: z$1.string(),
type: z$1.string(),
metadata: z$1.unknown(),
is_public: z$1.boolean(),
created_at: z$1.string(),
updated_at: z$1.string(),
federated_auth_method: z$1.string().nullable()
}).passthrough();
/**
* Schema for the full API response.
*/
const apiResponseSchema = z$1.object({ integrations: z$1.array(apiIntegrationSchema) }).passthrough();
/**
* Fetch integrations from the Deepnote API.
*
* Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike.
*
* @param baseUrl - The base URL of the Deepnote API
* @param token - The authentication token
* @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned.
* @returns Array of integrations from the API
* @throws ApiError if the request fails
*/
async function fetchIntegrations(baseUrl, token, integrationIds) {
const endpoint = new URL(`${baseUrl}/v2/integrations`);
endpoint.searchParams.set("includeMetadata", "true");
if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(","));
const url = endpoint.toString();
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json"
}
});
if (!response.ok) {
if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token.");
if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations.");
throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`);
}
const json = await response.json();
const parseResult = apiResponseSchema.safeParse(json);
if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`);
return parseResult.data.integrations;
}
//#endregion
//#region src/loading/integrations-document.ts
/**
* Parse integrations YAML content into a mutable `yaml` Document that preserves
* comments and formatting. Returns `null` for empty content.
*
* This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`.
*/
function parseIntegrationsDocument(content) {
if (!content.trim()) return null;
return parseDocument(content, {
strict: true,
version: "1.2"
});
}
/**
* Serialize an integrations Document back to YAML text, preserving comments and
* formatting and avoiding line wrapping.
*/
function serializeIntegrationsDocument(doc) {
return doc.toString({ lineWidth: 0 });
}
//#endregion
//#region src/loading/integrations-file-schema.ts
/**
* Schema for the integrations YAML file structure.
* Uses loose validation to accept any array of objects,
* allowing per-entry validation to report specific issues.
*
* Example file format:
* ```yaml
* integrations:
* - id: my-postgres
* name: My PostgreSQL
* type: pgsql
* metadata:
* host: localhost
* ...
* ```
*/
const baseIntegrationsFileSchema = z$1.object({ integrations: z$1.array(z$1.record(z$1.unknown())).optional().default([]) });
const integrationsFileSchema = z$1.object({ integrations: z$1.array(databaseIntegrationConfigSchema) });
//#endregion
//#region src/secret-field-paths.ts

@@ -909,2 +1114,3 @@ /**

spanner: ["service_account"],
"cloud-sql": ["service_account"],
mongodb: [

@@ -946,2 +1152,337 @@ "password",

//#endregion
export { AwsAuthMethods, BigQueryAuthMethods, BigQueryServiceAccountParseError, DatabaseAuthMethods, SnowflakeAuthMethods, SpannerServiceAccountParseError, TrinoAuthMethods, databaseIntegrationConfigSchema, databaseIntegrationTypes, databaseIntegrationTypesWithSslSupport, databaseMetadataSchemasByType, federatedAuthMethods, getEnvironmentVariablesForIntegrations, getSecretFieldPaths, getSnowflakeFederatedAuthSqlAlchemyInput, getSqlAlchemyInput, isDatabaseIntegrationType, isDatabaseIntegrationTypeWithSslSupport, isFederatedAuthMetadata, isFederatedAuthMethod, isSqlIntegrationType, sqlIntegrationTypes };
//#region src/loading/merge-integrations.ts
/**
* JSON schema URL for the integrations file.
* TODO - change to main branch when the PR is merged into main
*/
const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json";
/**
* Schema comment for the YAML file.
*/
const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`;
/**
* Error thrown when the integrations property has an invalid type.
*/
var InvalidIntegrationsTypeError = class extends Error {
constructor() {
super(`Invalid 'integrations' property: expected a list`);
this.name = "InvalidIntegrationsTypeError";
}
};
/**
* Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist.
* If the integrations property is missing or null, creates an empty array and returns a reference to it.
* Throws InvalidIntegrationsTypeError if integrations exists but is not an array.
*/
function getOrCreateIntegrationsFromDocument(doc) {
const integrations = doc.get("integrations");
if (integrations == null) {
const emptySeq = doc.createNode([]);
emptySeq.flow = false;
doc.set("integrations", emptySeq);
return emptySeq;
}
if (!isSeq(integrations)) throw new InvalidIntegrationsTypeError();
return integrations;
}
function getOrCreateIntegrationMetadata(doc, integrationMap) {
const existingMetadata = integrationMap.get("metadata", true);
if (isMap(existingMetadata)) return existingMetadata;
const metadata = doc.createNode({});
integrationMap.set("metadata", metadata);
return metadata;
}
function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) {
const secrets = {};
for (const [key, value] of Object.entries(integrationMetadata)) {
if (value === void 0) continue;
if (value === null) {
metadataMap.set(key, null);
continue;
}
if (!secretPaths.includes(key)) {
metadataMap.set(key, value);
continue;
}
const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key);
secrets[envVarName] = String(value);
metadataMap.set(key, createEnvVarRef(envVarName));
}
return secrets;
}
/**
* Update an existing integration entry in the document.
* Preserves any comments or extra fields while updating known fields.
* Extracts secrets and replaces them with env var references, preserving custom env var names.
*
* @returns Record of extracted secrets (envVarName -> secretValue)
*/
function updateIntegrationInDocument(doc, integrationMap, integration) {
const { id, name, type, metadata, federated_auth_method,...rest } = integration;
integrationMap.set("id", id);
integrationMap.set("name", name);
integrationMap.set("type", type);
const secretPaths = getSecretFieldPaths(type);
const secrets = updateIntegrationMetadataMap({
metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap),
integrationId: id,
integrationMetadata: metadata,
secretPaths
});
if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method);
else integrationMap.delete("federated_auth_method");
return secrets;
}
/**
* Add a new integration to the sequence.
* Extracts secrets and replaces them with env var references.
*
* @returns Record of extracted secrets (envVarName -> secretValue)
*/
function addIntegrationToSeq(doc, integrationsSeq, integration) {
const secretPaths = getSecretFieldPaths(integration.type);
const { metadata,...restIntegration } = integration;
const integrationMap = doc.createNode(restIntegration);
const metadataMap = doc.createNode({});
const secrets = updateIntegrationMetadataMap({
metadataMap,
integrationId: integration.id,
integrationMetadata: metadata,
secretPaths
});
integrationMap.setIn(["metadata"], metadataMap);
integrationsSeq.add(integrationMap);
return secrets;
}
/**
* Create a new Document with the schema comment.
*/
function createNewDocument() {
const doc = parseDocument("integrations: []\n", { version: "1.2" });
doc.commentBefore = SCHEMA_COMMENT;
const integrations = doc.get("integrations", true);
if (isSeq(integrations)) integrations.flow = false;
return doc;
}
/**
* Merge processed integrations into the document.
* Updates existing entries in-place (preserving comments) and adds new ones.
* Extracts secrets and replaces them with env var references during the merge.
* Preserves custom environment variable names from existing entries.
*
* @returns Object containing existing count and extracted secrets
*/
function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) {
const existingCount = integrationsSeq.items.length;
let updatedCount = 0;
let newCount = 0;
const secrets = {};
for (const databaseIntegration of processedIntegrations) {
const existingIntegration = integrationsSeq.items.find((item) => {
if (!isMap(item)) return false;
return item.get("id") === databaseIntegration.id;
});
if (existingIntegration != null) {
const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration);
Object.assign(secrets, extractedSecrets);
updatedCount++;
} else {
const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration);
Object.assign(secrets, extractedSecrets);
newCount++;
}
}
return {
secrets,
stats: {
existingCount,
newCount,
updatedCount
}
};
}
/**
* Error thrown when an API integration fails schema validation during conversion.
*/
var InvalidIntegrationError = class extends Error {
integrationId;
constructor(integrationId, message) {
super(message);
this.name = "InvalidIntegrationError";
this.integrationId = integrationId;
}
};
/**
* Convert API integrations to DatabaseIntegrationConfig.
* Validates each integration against the schema and collects errors for invalid ones.
*
* Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching).
*
* @param apiIntegrations - Integrations fetched from the API
* @returns Object containing validated integrations and any validation errors
*/
function convertApiIntegrations(apiIntegrations) {
const integrations = [];
const errors = [];
for (const apiIntegration of apiIntegrations) {
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
if (!config.success) {
errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API."));
continue;
}
integrations.push(config.data);
}
return {
integrations,
errors
};
}
/**
* Merge API integrations into an existing document (or create a new one).
* Extracts secrets and replaces them with env var references during the merge.
* Preserves custom environment variable names from existing entries.
* Invalid or unsupported integrations are silently skipped.
*
* @param doc - The YAML document to merge into
* @param apiIntegrations - Integrations fetched from the API
* @returns The extracted secrets and merge statistics
*/
function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) {
const integrationsSeq = getOrCreateIntegrationsFromDocument(doc);
const skipped = [];
return {
...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => {
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
if (!config.success) {
skipped.push({
id: apiIntegration.id,
name: apiIntegration.name,
type: apiIntegration.type,
issues: config.error.issues
});
return acc;
}
acc.push(config.data);
return acc;
}, [])),
skipped
};
}
//#endregion
//#region src/loading/merge-into-yaml.ts
/**
* Merge API integrations into existing integrations YAML content (or create a new
* document), returning the serialized YAML, the extracted secrets, and merge stats.
*
* This is a string-in / string-out convenience that mirrors what `deepnote
* integrations pull` does to the YAML file — usable without touching the `yaml`
* Document API. Callers persist `content` and `secrets` however they like (e.g.
* the CLI writes files; the VS Code extension can write via `workspace.fs`).
*
* @param existingContent - Current YAML content, or `null` if the file doesn't exist yet
* @param apiIntegrations - Integrations fetched from the API
*/
function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) {
const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument();
const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations);
if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT;
return {
content: serializeIntegrationsDocument(doc),
secrets,
stats,
skipped
};
}
//#endregion
//#region src/loading/parse-integrations.ts
/**
* Format Zod validation errors into ValidationIssue format.
*/
function formatZodIssues(issues, pathPrefix) {
return issues.map((issue) => ({
path: [pathPrefix, ...issue.path].filter(Boolean).join("."),
message: issue.message,
code: issue.code
}));
}
/**
* Parse integrations YAML content gracefully.
* Returns all valid integrations and collects validation issues for invalid ones.
*
* This is the content-accepting, environment-agnostic core (no filesystem or
* `process.env` access) — callers supply the YAML text and the variable map used
* to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this
* to read from disk and default the map to `process.env`.
*
* @returns Object containing valid integrations and any validation issues
*/
function parseIntegrations({ yaml, env = {} }) {
const emptyIntegrations = [];
const issues = [];
let parsed;
try {
parsed = parseYaml(yaml);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
issues.push({
path: "",
message: `Invalid YAML in integrations file: ${message}`,
code: "yaml_parse_error"
});
return {
integrations: emptyIntegrations,
issues
};
}
if (parsed === null || parsed === void 0) return {
integrations: emptyIntegrations,
issues
};
const fileResult = baseIntegrationsFileSchema.safeParse(parsed);
if (!fileResult.success) {
issues.push(...formatZodIssues(fileResult.error.issues, ""));
return {
integrations: emptyIntegrations,
issues
};
}
const entries = fileResult.data.integrations;
const integrations = [];
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
const pathPrefix = `integrations[${i}]`;
const entryId = z$1.string().safeParse(entry.id).data;
const integrationLabel = z$1.string().safeParse(entry.name).data || entryId;
let resolvedEntry;
try {
resolvedEntry = resolveEnvVarRefsFromMap(entry, env);
} catch (error) {
if (error instanceof EnvVarResolutionError) {
const context = integrationLabel ? `Integration "${integrationLabel}": ` : "";
issues.push({
path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix,
message: `${context}${error.message}`,
code: "env_var_not_defined"
});
continue;
}
throw error;
}
const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry);
if (result.success) integrations.push(result.data);
else {
const formattedIssues = formatZodIssues(result.error.issues, pathPrefix);
if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`;
issues.push(...formattedIssues);
}
}
return {
integrations,
issues
};
}
//#endregion
export { ApiError, AwsAuthMethods, BUILTIN_INTEGRATIONS, BigQueryAuthMethods, BigQueryServiceAccountParseError, DEFAULT_API_URL, DEFAULT_ENV_FILE, DEFAULT_INTEGRATIONS_FILE, DatabaseAuthMethods, ENV_VAR_REF_PREFIX, EnvVarResolutionError, InvalidIntegrationError, InvalidIntegrationsTypeError, JSON_SCHEMA_URL, SCHEMA_COMMENT, SnowflakeAuthMethods, SpannerServiceAccountParseError, TrinoAuthMethods, addIntegrationToSeq, apiIntegrationSchema, apiResponseSchema, baseIntegrationsFileSchema, convertApiIntegrations, createEnvVarRef, createNewDocument, databaseIntegrationConfigSchema, databaseIntegrationTypes, databaseIntegrationTypesWithSslSupport, databaseMetadataSchemasByType, extractEnvVarName, federatedAuthMethods, fetchIntegrations, generateEnvVarName, getEnvironmentVariablesForIntegrations, getOrCreateIntegrationMetadata, getOrCreateIntegrationsFromDocument, getSecretFieldPaths, getSnowflakeFederatedAuthSqlAlchemyInput, getSqlAlchemyInput, getSqlEnvVarName, integrationsFileSchema, isDatabaseIntegrationType, isDatabaseIntegrationTypeWithSslSupport, isEnvVarRef, isFederatedAuthMetadata, isFederatedAuthMethod, isSqlIntegrationType, mergeApiIntegrationsIntoDocument, mergeApiIntegrationsIntoYaml, mergeProcessedIntegrations, parseEnvVarRef, parseIntegrations, parseIntegrationsDocument, resolveEnvVarRefsFromMap, serializeIntegrationsDocument, sqlIntegrationTypes, updateIntegrationInDocument, updateIntegrationMetadataMap };
{
"name": "@deepnote/database-integrations",
"version": "1.4.3",
"version": "1.5.0",
"description": "Deepnote database integration definitions",

@@ -30,3 +30,5 @@ "keywords": [],

"dependencies": {
"zod": "3.25.76"
"yaml": "^2.8.3",
"zod": "3.25.76",
"@deepnote/blocks": "4.6.0"
},

@@ -33,0 +35,0 @@ "publishConfig": {

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display