@deepnote/database-integrations
Advanced tools
+580
-1
@@ -26,2 +26,6 @@ //#region rolldown:runtime | ||
| zod = __toESM(zod); | ||
| let yaml = require("yaml"); | ||
| yaml = __toESM(yaml); | ||
| let __deepnote_blocks = require("@deepnote/blocks"); | ||
| __deepnote_blocks = __toESM(__deepnote_blocks); | ||
@@ -101,2 +105,3 @@ //#region src/sql-integration-auth-methods.ts | ||
| }); | ||
| const cloudSqlMetadataSchema = zod.z.object({ service_account: zod.z.string() }); | ||
| const databricksMetadataSchema = zod.z.object({ | ||
@@ -303,2 +308,3 @@ host: zod.z.string(), | ||
| "clickhouse": clickhouseMetadataSchema, | ||
| "cloud-sql": cloudSqlMetadataSchema, | ||
| "databricks": databricksMetadataSchema, | ||
@@ -348,2 +354,7 @@ "dremio": dremioMetadataSchema, | ||
| commonIntegrationConfig.extend({ | ||
| type: zod.default.literal("cloud-sql"), | ||
| metadata: databaseMetadataSchemasByType["cloud-sql"], | ||
| federated_auth_method: zod.default.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: zod.default.literal("databricks"), | ||
@@ -574,2 +585,3 @@ metadata: databaseMetadataSchemasByType.databricks, | ||
| case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata); | ||
| case "cloud-sql": return null; | ||
| case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata); | ||
@@ -869,2 +881,3 @@ case "dremio": return getDremioSqlAlchemyInput(integration.metadata); | ||
| "clickhouse", | ||
| "cloud-sql", | ||
| "databricks", | ||
@@ -908,2 +921,196 @@ "dremio", | ||
| //#endregion | ||
| //#region src/loading/api-error.ts | ||
| /** | ||
| * Error thrown when a Deepnote API request fails. | ||
| */ | ||
| var ApiError = class extends Error { | ||
| statusCode; | ||
| constructor(statusCode, message) { | ||
| super(message); | ||
| this.name = "ApiError"; | ||
| this.statusCode = statusCode; | ||
| } | ||
| }; | ||
| //#endregion | ||
| //#region src/loading/constants.ts | ||
| /** Built-in integrations that don't require external configuration. */ | ||
| const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]); | ||
| /** Default `.env` file name for storing integration secrets. */ | ||
| const DEFAULT_ENV_FILE = ".env"; | ||
| /** Default integrations file name. */ | ||
| const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml"; | ||
| /** Default Deepnote API base URL. */ | ||
| const DEFAULT_API_URL = "https://api.deepnote.com"; | ||
| //#endregion | ||
| //#region src/loading/env-var-refs.ts | ||
| const ENV_VAR_REF_PREFIX = "env:"; | ||
| function generateEnvVarName(integrationId, fieldPath) { | ||
| return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`; | ||
| } | ||
| function parseEnvVarRef(value) { | ||
| if (typeof value !== "string") return null; | ||
| if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null; | ||
| const varName = value.slice(4); | ||
| if (!varName) return null; | ||
| return { | ||
| prefix: "env", | ||
| varName | ||
| }; | ||
| } | ||
| function createEnvVarRef(varName) { | ||
| return `${ENV_VAR_REF_PREFIX}${varName}`; | ||
| } | ||
| function isEnvVarRef(value) { | ||
| return typeof value === "string" && parseEnvVarRef(value) !== null; | ||
| } | ||
| function extractEnvVarName(value) { | ||
| if (typeof value !== "string") return null; | ||
| return parseEnvVarRef(value)?.varName ?? null; | ||
| } | ||
| /** | ||
| * Error thrown when an environment variable reference cannot be resolved. | ||
| */ | ||
| var EnvVarResolutionError = class extends Error { | ||
| varName; | ||
| path; | ||
| constructor(varName, path) { | ||
| const pathInfo = path ? ` at "${path}"` : ""; | ||
| super(`Environment variable "${varName}" is not defined${pathInfo}`); | ||
| this.name = "EnvVarResolutionError"; | ||
| this.varName = varName; | ||
| this.path = path; | ||
| } | ||
| }; | ||
| /** | ||
| * Recursively resolves environment variable references in an object using an explicit variable map. | ||
| * Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map. | ||
| * | ||
| * @param obj - The object to process (can be any value: primitive, array, or object) | ||
| * @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store) | ||
| * @param currentPath - Internal parameter for tracking the current path in the object (for error messages) | ||
| * @returns A new object with all env var references resolved | ||
| * @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved | ||
| */ | ||
| function resolveEnvVarRefsFromMap(obj, vars, currentPath) { | ||
| if (obj === null || obj === void 0) return obj; | ||
| if (typeof obj === "string") { | ||
| const parsed = parseEnvVarRef(obj); | ||
| if (parsed) { | ||
| const envValue = vars[parsed.varName]; | ||
| if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath); | ||
| return envValue; | ||
| } | ||
| return obj; | ||
| } | ||
| if (Array.isArray(obj)) return obj.map((item, index) => { | ||
| return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`); | ||
| }); | ||
| if (typeof obj === "object") { | ||
| const result = {}; | ||
| for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key); | ||
| return result; | ||
| } | ||
| return obj; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/fetch-integrations.ts | ||
| /** | ||
| * Schema for a single integration from the API. | ||
| */ | ||
| const apiIntegrationSchema = zod.z.object({ | ||
| id: zod.z.string(), | ||
| name: zod.z.string(), | ||
| type: zod.z.string(), | ||
| metadata: zod.z.unknown(), | ||
| is_public: zod.z.boolean(), | ||
| created_at: zod.z.string(), | ||
| updated_at: zod.z.string(), | ||
| federated_auth_method: zod.z.string().nullable() | ||
| }).passthrough(); | ||
| /** | ||
| * Schema for the full API response. | ||
| */ | ||
| const apiResponseSchema = zod.z.object({ integrations: zod.z.array(apiIntegrationSchema) }).passthrough(); | ||
| /** | ||
| * Fetch integrations from the Deepnote API. | ||
| * | ||
| * Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike. | ||
| * | ||
| * @param baseUrl - The base URL of the Deepnote API | ||
| * @param token - The authentication token | ||
| * @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned. | ||
| * @returns Array of integrations from the API | ||
| * @throws ApiError if the request fails | ||
| */ | ||
| async function fetchIntegrations(baseUrl, token, integrationIds) { | ||
| const endpoint = new URL(`${baseUrl}/v2/integrations`); | ||
| endpoint.searchParams.set("includeMetadata", "true"); | ||
| if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(",")); | ||
| const url = endpoint.toString(); | ||
| const response = await fetch(url, { | ||
| method: "GET", | ||
| headers: { | ||
| Authorization: `Bearer ${token}`, | ||
| "Content-Type": "application/json" | ||
| } | ||
| }); | ||
| if (!response.ok) { | ||
| if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token."); | ||
| if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations."); | ||
| throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`); | ||
| } | ||
| const json = await response.json(); | ||
| const parseResult = apiResponseSchema.safeParse(json); | ||
| if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`); | ||
| return parseResult.data.integrations; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/integrations-document.ts | ||
| /** | ||
| * Parse integrations YAML content into a mutable `yaml` Document that preserves | ||
| * comments and formatting. Returns `null` for empty content. | ||
| * | ||
| * This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`. | ||
| */ | ||
| function parseIntegrationsDocument(content) { | ||
| if (!content.trim()) return null; | ||
| return (0, yaml.parseDocument)(content, { | ||
| strict: true, | ||
| version: "1.2" | ||
| }); | ||
| } | ||
| /** | ||
| * Serialize an integrations Document back to YAML text, preserving comments and | ||
| * formatting and avoiding line wrapping. | ||
| */ | ||
| function serializeIntegrationsDocument(doc) { | ||
| return doc.toString({ lineWidth: 0 }); | ||
| } | ||
| //#endregion | ||
| //#region src/loading/integrations-file-schema.ts | ||
| /** | ||
| * Schema for the integrations YAML file structure. | ||
| * Uses loose validation to accept any array of objects, | ||
| * allowing per-entry validation to report specific issues. | ||
| * | ||
| * Example file format: | ||
| * ```yaml | ||
| * integrations: | ||
| * - id: my-postgres | ||
| * name: My PostgreSQL | ||
| * type: pgsql | ||
| * metadata: | ||
| * host: localhost | ||
| * ... | ||
| * ``` | ||
| */ | ||
| const baseIntegrationsFileSchema = zod.z.object({ integrations: zod.z.array(zod.z.record(zod.z.unknown())).optional().default([]) }); | ||
| const integrationsFileSchema = zod.z.object({ integrations: zod.z.array(databaseIntegrationConfigSchema) }); | ||
| //#endregion | ||
| //#region src/secret-field-paths.ts | ||
@@ -930,2 +1137,3 @@ /** | ||
| spanner: ["service_account"], | ||
| "cloud-sql": ["service_account"], | ||
| mongodb: [ | ||
@@ -967,9 +1175,362 @@ "password", | ||
| //#endregion | ||
| //#region src/loading/merge-integrations.ts | ||
| /** | ||
| * JSON schema URL for the integrations file. | ||
| * TODO - change to main branch when the PR is merged into main | ||
| */ | ||
| const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json"; | ||
| /** | ||
| * Schema comment for the YAML file. | ||
| */ | ||
| const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`; | ||
| /** | ||
| * Error thrown when the integrations property has an invalid type. | ||
| */ | ||
| var InvalidIntegrationsTypeError = class extends Error { | ||
| constructor() { | ||
| super(`Invalid 'integrations' property: expected a list`); | ||
| this.name = "InvalidIntegrationsTypeError"; | ||
| } | ||
| }; | ||
| /** | ||
| * Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist. | ||
| * If the integrations property is missing or null, creates an empty array and returns a reference to it. | ||
| * Throws InvalidIntegrationsTypeError if integrations exists but is not an array. | ||
| */ | ||
| function getOrCreateIntegrationsFromDocument(doc) { | ||
| const integrations = doc.get("integrations"); | ||
| if (integrations == null) { | ||
| const emptySeq = doc.createNode([]); | ||
| emptySeq.flow = false; | ||
| doc.set("integrations", emptySeq); | ||
| return emptySeq; | ||
| } | ||
| if (!(0, yaml.isSeq)(integrations)) throw new InvalidIntegrationsTypeError(); | ||
| return integrations; | ||
| } | ||
| function getOrCreateIntegrationMetadata(doc, integrationMap) { | ||
| const existingMetadata = integrationMap.get("metadata", true); | ||
| if ((0, yaml.isMap)(existingMetadata)) return existingMetadata; | ||
| const metadata = doc.createNode({}); | ||
| integrationMap.set("metadata", metadata); | ||
| return metadata; | ||
| } | ||
| function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) { | ||
| const secrets = {}; | ||
| for (const [key, value] of Object.entries(integrationMetadata)) { | ||
| if (value === void 0) continue; | ||
| if (value === null) { | ||
| metadataMap.set(key, null); | ||
| continue; | ||
| } | ||
| if (!secretPaths.includes(key)) { | ||
| metadataMap.set(key, value); | ||
| continue; | ||
| } | ||
| const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key); | ||
| secrets[envVarName] = String(value); | ||
| metadataMap.set(key, createEnvVarRef(envVarName)); | ||
| } | ||
| return secrets; | ||
| } | ||
| /** | ||
| * Update an existing integration entry in the document. | ||
| * Preserves any comments or extra fields while updating known fields. | ||
| * Extracts secrets and replaces them with env var references, preserving custom env var names. | ||
| * | ||
| * @returns Record of extracted secrets (envVarName -> secretValue) | ||
| */ | ||
| function updateIntegrationInDocument(doc, integrationMap, integration) { | ||
| const { id, name, type, metadata, federated_auth_method,...rest } = integration; | ||
| integrationMap.set("id", id); | ||
| integrationMap.set("name", name); | ||
| integrationMap.set("type", type); | ||
| const secretPaths = getSecretFieldPaths(type); | ||
| const secrets = updateIntegrationMetadataMap({ | ||
| metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap), | ||
| integrationId: id, | ||
| integrationMetadata: metadata, | ||
| secretPaths | ||
| }); | ||
| if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method); | ||
| else integrationMap.delete("federated_auth_method"); | ||
| return secrets; | ||
| } | ||
| /** | ||
| * Add a new integration to the sequence. | ||
| * Extracts secrets and replaces them with env var references. | ||
| * | ||
| * @returns Record of extracted secrets (envVarName -> secretValue) | ||
| */ | ||
| function addIntegrationToSeq(doc, integrationsSeq, integration) { | ||
| const secretPaths = getSecretFieldPaths(integration.type); | ||
| const { metadata,...restIntegration } = integration; | ||
| const integrationMap = doc.createNode(restIntegration); | ||
| const metadataMap = doc.createNode({}); | ||
| const secrets = updateIntegrationMetadataMap({ | ||
| metadataMap, | ||
| integrationId: integration.id, | ||
| integrationMetadata: metadata, | ||
| secretPaths | ||
| }); | ||
| integrationMap.setIn(["metadata"], metadataMap); | ||
| integrationsSeq.add(integrationMap); | ||
| return secrets; | ||
| } | ||
| /** | ||
| * Create a new Document with the schema comment. | ||
| */ | ||
| function createNewDocument() { | ||
| const doc = (0, yaml.parseDocument)("integrations: []\n", { version: "1.2" }); | ||
| doc.commentBefore = SCHEMA_COMMENT; | ||
| const integrations = doc.get("integrations", true); | ||
| if ((0, yaml.isSeq)(integrations)) integrations.flow = false; | ||
| return doc; | ||
| } | ||
| /** | ||
| * Merge processed integrations into the document. | ||
| * Updates existing entries in-place (preserving comments) and adds new ones. | ||
| * Extracts secrets and replaces them with env var references during the merge. | ||
| * Preserves custom environment variable names from existing entries. | ||
| * | ||
| * @returns Object containing existing count and extracted secrets | ||
| */ | ||
| function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) { | ||
| const existingCount = integrationsSeq.items.length; | ||
| let updatedCount = 0; | ||
| let newCount = 0; | ||
| const secrets = {}; | ||
| for (const databaseIntegration of processedIntegrations) { | ||
| const existingIntegration = integrationsSeq.items.find((item) => { | ||
| if (!(0, yaml.isMap)(item)) return false; | ||
| return item.get("id") === databaseIntegration.id; | ||
| }); | ||
| if (existingIntegration != null) { | ||
| const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration); | ||
| Object.assign(secrets, extractedSecrets); | ||
| updatedCount++; | ||
| } else { | ||
| const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration); | ||
| Object.assign(secrets, extractedSecrets); | ||
| newCount++; | ||
| } | ||
| } | ||
| return { | ||
| secrets, | ||
| stats: { | ||
| existingCount, | ||
| newCount, | ||
| updatedCount | ||
| } | ||
| }; | ||
| } | ||
| /** | ||
| * Error thrown when an API integration fails schema validation during conversion. | ||
| */ | ||
| var InvalidIntegrationError = class extends Error { | ||
| integrationId; | ||
| constructor(integrationId, message) { | ||
| super(message); | ||
| this.name = "InvalidIntegrationError"; | ||
| this.integrationId = integrationId; | ||
| } | ||
| }; | ||
| /** | ||
| * Convert API integrations to DatabaseIntegrationConfig. | ||
| * Validates each integration against the schema and collects errors for invalid ones. | ||
| * | ||
| * Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching). | ||
| * | ||
| * @param apiIntegrations - Integrations fetched from the API | ||
| * @returns Object containing validated integrations and any validation errors | ||
| */ | ||
| function convertApiIntegrations(apiIntegrations) { | ||
| const integrations = []; | ||
| const errors = []; | ||
| for (const apiIntegration of apiIntegrations) { | ||
| const config = databaseIntegrationConfigSchema.safeParse(apiIntegration); | ||
| if (!config.success) { | ||
| errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API.")); | ||
| continue; | ||
| } | ||
| integrations.push(config.data); | ||
| } | ||
| return { | ||
| integrations, | ||
| errors | ||
| }; | ||
| } | ||
| /** | ||
| * Merge API integrations into an existing document (or create a new one). | ||
| * Extracts secrets and replaces them with env var references during the merge. | ||
| * Preserves custom environment variable names from existing entries. | ||
| * Invalid or unsupported integrations are silently skipped. | ||
| * | ||
| * @param doc - The YAML document to merge into | ||
| * @param apiIntegrations - Integrations fetched from the API | ||
| * @returns The extracted secrets and merge statistics | ||
| */ | ||
| function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) { | ||
| const integrationsSeq = getOrCreateIntegrationsFromDocument(doc); | ||
| const skipped = []; | ||
| return { | ||
| ...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => { | ||
| const config = databaseIntegrationConfigSchema.safeParse(apiIntegration); | ||
| if (!config.success) { | ||
| skipped.push({ | ||
| id: apiIntegration.id, | ||
| name: apiIntegration.name, | ||
| type: apiIntegration.type, | ||
| issues: config.error.issues | ||
| }); | ||
| return acc; | ||
| } | ||
| acc.push(config.data); | ||
| return acc; | ||
| }, [])), | ||
| skipped | ||
| }; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/merge-into-yaml.ts | ||
| /** | ||
| * Merge API integrations into existing integrations YAML content (or create a new | ||
| * document), returning the serialized YAML, the extracted secrets, and merge stats. | ||
| * | ||
| * This is a string-in / string-out convenience that mirrors what `deepnote | ||
| * integrations pull` does to the YAML file — usable without touching the `yaml` | ||
| * Document API. Callers persist `content` and `secrets` however they like (e.g. | ||
| * the CLI writes files; the VS Code extension can write via `workspace.fs`). | ||
| * | ||
| * @param existingContent - Current YAML content, or `null` if the file doesn't exist yet | ||
| * @param apiIntegrations - Integrations fetched from the API | ||
| */ | ||
| function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) { | ||
| const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument(); | ||
| const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations); | ||
| if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT; | ||
| return { | ||
| content: serializeIntegrationsDocument(doc), | ||
| secrets, | ||
| stats, | ||
| skipped | ||
| }; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/parse-integrations.ts | ||
| /** | ||
| * Format Zod validation errors into ValidationIssue format. | ||
| */ | ||
| function formatZodIssues(issues, pathPrefix) { | ||
| return issues.map((issue) => ({ | ||
| path: [pathPrefix, ...issue.path].filter(Boolean).join("."), | ||
| message: issue.message, | ||
| code: issue.code | ||
| })); | ||
| } | ||
| /** | ||
| * Parse integrations YAML content gracefully. | ||
| * Returns all valid integrations and collects validation issues for invalid ones. | ||
| * | ||
| * This is the content-accepting, environment-agnostic core (no filesystem or | ||
| * `process.env` access) — callers supply the YAML text and the variable map used | ||
| * to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this | ||
| * to read from disk and default the map to `process.env`. | ||
| * | ||
| * @returns Object containing valid integrations and any validation issues | ||
| */ | ||
| function parseIntegrations({ yaml: yaml$1, env = {} }) { | ||
| const emptyIntegrations = []; | ||
| const issues = []; | ||
| let parsed; | ||
| try { | ||
| parsed = (0, __deepnote_blocks.parseYaml)(yaml$1); | ||
| } catch (error) { | ||
| const message = error instanceof Error ? error.message : String(error); | ||
| issues.push({ | ||
| path: "", | ||
| message: `Invalid YAML in integrations file: ${message}`, | ||
| code: "yaml_parse_error" | ||
| }); | ||
| return { | ||
| integrations: emptyIntegrations, | ||
| issues | ||
| }; | ||
| } | ||
| if (parsed === null || parsed === void 0) return { | ||
| integrations: emptyIntegrations, | ||
| issues | ||
| }; | ||
| const fileResult = baseIntegrationsFileSchema.safeParse(parsed); | ||
| if (!fileResult.success) { | ||
| issues.push(...formatZodIssues(fileResult.error.issues, "")); | ||
| return { | ||
| integrations: emptyIntegrations, | ||
| issues | ||
| }; | ||
| } | ||
| const entries = fileResult.data.integrations; | ||
| const integrations = []; | ||
| for (let i = 0; i < entries.length; i++) { | ||
| const entry = entries[i]; | ||
| const pathPrefix = `integrations[${i}]`; | ||
| const entryId = zod.z.string().safeParse(entry.id).data; | ||
| const integrationLabel = zod.z.string().safeParse(entry.name).data || entryId; | ||
| let resolvedEntry; | ||
| try { | ||
| resolvedEntry = resolveEnvVarRefsFromMap(entry, env); | ||
| } catch (error) { | ||
| if (error instanceof EnvVarResolutionError) { | ||
| const context = integrationLabel ? `Integration "${integrationLabel}": ` : ""; | ||
| issues.push({ | ||
| path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix, | ||
| message: `${context}${error.message}`, | ||
| code: "env_var_not_defined" | ||
| }); | ||
| continue; | ||
| } | ||
| throw error; | ||
| } | ||
| const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry); | ||
| if (result.success) integrations.push(result.data); | ||
| else { | ||
| const formattedIssues = formatZodIssues(result.error.issues, pathPrefix); | ||
| if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`; | ||
| issues.push(...formattedIssues); | ||
| } | ||
| } | ||
| return { | ||
| integrations, | ||
| issues | ||
| }; | ||
| } | ||
| //#endregion | ||
| exports.ApiError = ApiError; | ||
| exports.AwsAuthMethods = AwsAuthMethods; | ||
| exports.BUILTIN_INTEGRATIONS = BUILTIN_INTEGRATIONS; | ||
| exports.BigQueryAuthMethods = BigQueryAuthMethods; | ||
| exports.BigQueryServiceAccountParseError = BigQueryServiceAccountParseError; | ||
| exports.DEFAULT_API_URL = DEFAULT_API_URL; | ||
| exports.DEFAULT_ENV_FILE = DEFAULT_ENV_FILE; | ||
| exports.DEFAULT_INTEGRATIONS_FILE = DEFAULT_INTEGRATIONS_FILE; | ||
| exports.DatabaseAuthMethods = DatabaseAuthMethods; | ||
| exports.ENV_VAR_REF_PREFIX = ENV_VAR_REF_PREFIX; | ||
| exports.EnvVarResolutionError = EnvVarResolutionError; | ||
| exports.InvalidIntegrationError = InvalidIntegrationError; | ||
| exports.InvalidIntegrationsTypeError = InvalidIntegrationsTypeError; | ||
| exports.JSON_SCHEMA_URL = JSON_SCHEMA_URL; | ||
| exports.SCHEMA_COMMENT = SCHEMA_COMMENT; | ||
| exports.SnowflakeAuthMethods = SnowflakeAuthMethods; | ||
| exports.SpannerServiceAccountParseError = SpannerServiceAccountParseError; | ||
| exports.TrinoAuthMethods = TrinoAuthMethods; | ||
| exports.addIntegrationToSeq = addIntegrationToSeq; | ||
| exports.apiIntegrationSchema = apiIntegrationSchema; | ||
| exports.apiResponseSchema = apiResponseSchema; | ||
| exports.baseIntegrationsFileSchema = baseIntegrationsFileSchema; | ||
| exports.convertApiIntegrations = convertApiIntegrations; | ||
| exports.createEnvVarRef = createEnvVarRef; | ||
| exports.createNewDocument = createNewDocument; | ||
| exports.databaseIntegrationConfigSchema = databaseIntegrationConfigSchema; | ||
@@ -979,12 +1540,30 @@ exports.databaseIntegrationTypes = databaseIntegrationTypes; | ||
| exports.databaseMetadataSchemasByType = databaseMetadataSchemasByType; | ||
| exports.extractEnvVarName = extractEnvVarName; | ||
| exports.federatedAuthMethods = federatedAuthMethods; | ||
| exports.fetchIntegrations = fetchIntegrations; | ||
| exports.generateEnvVarName = generateEnvVarName; | ||
| exports.getEnvironmentVariablesForIntegrations = getEnvironmentVariablesForIntegrations; | ||
| exports.getOrCreateIntegrationMetadata = getOrCreateIntegrationMetadata; | ||
| exports.getOrCreateIntegrationsFromDocument = getOrCreateIntegrationsFromDocument; | ||
| exports.getSecretFieldPaths = getSecretFieldPaths; | ||
| exports.getSnowflakeFederatedAuthSqlAlchemyInput = getSnowflakeFederatedAuthSqlAlchemyInput; | ||
| exports.getSqlAlchemyInput = getSqlAlchemyInput; | ||
| exports.getSqlEnvVarName = getSqlEnvVarName; | ||
| exports.integrationsFileSchema = integrationsFileSchema; | ||
| exports.isDatabaseIntegrationType = isDatabaseIntegrationType; | ||
| exports.isDatabaseIntegrationTypeWithSslSupport = isDatabaseIntegrationTypeWithSslSupport; | ||
| exports.isEnvVarRef = isEnvVarRef; | ||
| exports.isFederatedAuthMetadata = isFederatedAuthMetadata; | ||
| exports.isFederatedAuthMethod = isFederatedAuthMethod; | ||
| exports.isSqlIntegrationType = isSqlIntegrationType; | ||
| exports.sqlIntegrationTypes = sqlIntegrationTypes; | ||
| exports.mergeApiIntegrationsIntoDocument = mergeApiIntegrationsIntoDocument; | ||
| exports.mergeApiIntegrationsIntoYaml = mergeApiIntegrationsIntoYaml; | ||
| exports.mergeProcessedIntegrations = mergeProcessedIntegrations; | ||
| exports.parseEnvVarRef = parseEnvVarRef; | ||
| exports.parseIntegrations = parseIntegrations; | ||
| exports.parseIntegrationsDocument = parseIntegrationsDocument; | ||
| exports.resolveEnvVarRefsFromMap = resolveEnvVarRefsFromMap; | ||
| exports.serializeIntegrationsDocument = serializeIntegrationsDocument; | ||
| exports.sqlIntegrationTypes = sqlIntegrationTypes; | ||
| exports.updateIntegrationInDocument = updateIntegrationInDocument; | ||
| exports.updateIntegrationMetadataMap = updateIntegrationMetadataMap; |
+758
-217
@@ -1,2 +0,4 @@ | ||
| import z$1, { z } from "zod"; | ||
| import z, { z as z$1 } from "zod"; | ||
| import { isMap, isSeq, parseDocument } from "yaml"; | ||
| import { parseYaml } from "@deepnote/blocks"; | ||
@@ -46,86 +48,87 @@ //#region src/sql-integration-auth-methods.ts | ||
| //#region src/database-integration-metadata-schemas.ts | ||
| const athenaMetadataSchema = z.object({ | ||
| access_key_id: z.string(), | ||
| region: z.string(), | ||
| s3_output_path: z.string(), | ||
| secret_access_key: z.string(), | ||
| workgroup: z.string().optional() | ||
| const athenaMetadataSchema = z$1.object({ | ||
| access_key_id: z$1.string(), | ||
| region: z$1.string(), | ||
| s3_output_path: z$1.string(), | ||
| secret_access_key: z$1.string(), | ||
| workgroup: z$1.string().optional() | ||
| }); | ||
| const bigqueryMetadataSchema = z.discriminatedUnion("authMethod", [z.object({ | ||
| authMethod: z.literal(BigQueryAuthMethods.ServiceAccount).optional(), | ||
| service_account: z.string() | ||
| }), z.object({ | ||
| authMethod: z.literal(BigQueryAuthMethods.GoogleOauth), | ||
| project: z.string(), | ||
| clientId: z.string(), | ||
| clientSecret: z.string() | ||
| const bigqueryMetadataSchema = z$1.discriminatedUnion("authMethod", [z$1.object({ | ||
| authMethod: z$1.literal(BigQueryAuthMethods.ServiceAccount).optional(), | ||
| service_account: z$1.string() | ||
| }), z$1.object({ | ||
| authMethod: z$1.literal(BigQueryAuthMethods.GoogleOauth), | ||
| project: z$1.string(), | ||
| clientId: z$1.string(), | ||
| clientSecret: z$1.string() | ||
| })]); | ||
| const clickhouseMetadataSchema = z.object({ | ||
| host: z.string(), | ||
| user: z.string(), | ||
| password: z.string().optional(), | ||
| database: z.string(), | ||
| port: z.string().optional(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| const clickhouseMetadataSchema = z$1.object({ | ||
| host: z$1.string(), | ||
| user: z$1.string(), | ||
| password: z$1.string().optional(), | ||
| database: z$1.string(), | ||
| port: z$1.string().optional(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const databricksMetadataSchema = z.object({ | ||
| host: z.string(), | ||
| httpPath: z.string(), | ||
| token: z.string(), | ||
| port: z.string(), | ||
| schema: z.string().optional(), | ||
| catalog: z.string().optional(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional() | ||
| const cloudSqlMetadataSchema = z$1.object({ service_account: z$1.string() }); | ||
| const databricksMetadataSchema = z$1.object({ | ||
| host: z$1.string(), | ||
| httpPath: z$1.string(), | ||
| token: z$1.string(), | ||
| port: z$1.string(), | ||
| schema: z$1.string().optional(), | ||
| catalog: z$1.string().optional(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional() | ||
| }); | ||
| const dremioMetadataSchema = z.object({ | ||
| schema: z.string(), | ||
| host: z.string(), | ||
| port: z.string(), | ||
| token: z.string(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional() | ||
| const dremioMetadataSchema = z$1.object({ | ||
| schema: z$1.string(), | ||
| host: z$1.string(), | ||
| port: z$1.string(), | ||
| token: z$1.string(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional() | ||
| }); | ||
| const mongodbMetadataSchema = z.object({ | ||
| connection_string: z.string(), | ||
| rawConnectionString: z.string().optional(), | ||
| prefix: z.string().optional(), | ||
| host: z.string().optional(), | ||
| port: z.string().optional(), | ||
| user: z.string().optional(), | ||
| password: z.string().optional(), | ||
| database: z.string().optional(), | ||
| options: z.string().optional(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| const mongodbMetadataSchema = z$1.object({ | ||
| connection_string: z$1.string(), | ||
| rawConnectionString: z$1.string().optional(), | ||
| prefix: z$1.string().optional(), | ||
| host: z$1.string().optional(), | ||
| port: z$1.string().optional(), | ||
| user: z$1.string().optional(), | ||
| password: z$1.string().optional(), | ||
| database: z$1.string().optional(), | ||
| options: z$1.string().optional(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const pandasDataframeMetadataSchema = z.object({}); | ||
| const commonRedshiftMetadataSchema = z.object({ | ||
| database: z.string(), | ||
| host: z.string(), | ||
| port: z.string().optional(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| const pandasDataframeMetadataSchema = z$1.object({}); | ||
| const commonRedshiftMetadataSchema = z$1.object({ | ||
| database: z$1.string(), | ||
| host: z$1.string(), | ||
| port: z$1.string().optional(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const redshiftMetadataSchema = z.preprocess((data) => { | ||
| const redshiftMetadataSchema = z$1.preprocess((data) => { | ||
| if (typeof data === "object" && data !== null && (!("authMethod" in data) || !data.authMethod)) return { | ||
@@ -136,27 +139,27 @@ ...data, | ||
| return data; | ||
| }, z.union([ | ||
| }, z$1.union([ | ||
| commonRedshiftMetadataSchema.extend({ | ||
| authMethod: z.literal(DatabaseAuthMethods.UsernameAndPassword).optional(), | ||
| user: z.string(), | ||
| password: z.string() | ||
| authMethod: z$1.literal(DatabaseAuthMethods.UsernameAndPassword).optional(), | ||
| user: z$1.string(), | ||
| password: z$1.string() | ||
| }), | ||
| commonRedshiftMetadataSchema.extend({ | ||
| authMethod: z.literal(AwsAuthMethods.IamRole), | ||
| roleArn: z.string(), | ||
| roleExternalId: z.string(), | ||
| roleNonce: z.string() | ||
| authMethod: z$1.literal(AwsAuthMethods.IamRole), | ||
| roleArn: z$1.string(), | ||
| roleExternalId: z$1.string(), | ||
| roleNonce: z$1.string() | ||
| }), | ||
| commonRedshiftMetadataSchema.extend({ authMethod: z.literal(DatabaseAuthMethods.IndividualCredentials) }) | ||
| commonRedshiftMetadataSchema.extend({ authMethod: z$1.literal(DatabaseAuthMethods.IndividualCredentials) }) | ||
| ])); | ||
| const commonSnowflakeMetadataSchema = z.object({ | ||
| accountName: z.string(), | ||
| warehouse: z.string().optional(), | ||
| database: z.string().optional(), | ||
| role: z.string().optional(), | ||
| dbt: z.boolean().optional(), | ||
| dbtServiceToken: z.string().optional(), | ||
| dbtPrimaryJobId: z.string().optional(), | ||
| dbtProxyServerUrl: z.string().optional() | ||
| const commonSnowflakeMetadataSchema = z$1.object({ | ||
| accountName: z$1.string(), | ||
| warehouse: z$1.string().optional(), | ||
| database: z$1.string().optional(), | ||
| role: z$1.string().optional(), | ||
| dbt: z$1.boolean().optional(), | ||
| dbtServiceToken: z$1.string().optional(), | ||
| dbtPrimaryJobId: z$1.string().optional(), | ||
| dbtProxyServerUrl: z$1.string().optional() | ||
| }); | ||
| const snowflakeMetadataSchema = z.preprocess((data) => { | ||
| const snowflakeMetadataSchema = z$1.preprocess((data) => { | ||
| if (typeof data === "object" && data !== null && (!("authMethod" in data) || !data.authMethod)) return { | ||
@@ -167,109 +170,109 @@ ...data, | ||
| return data; | ||
| }, z.discriminatedUnion("authMethod", [ | ||
| }, z$1.discriminatedUnion("authMethod", [ | ||
| commonSnowflakeMetadataSchema.extend({ | ||
| authMethod: z.literal(SnowflakeAuthMethods.Password).optional(), | ||
| username: z.string(), | ||
| password: z.string() | ||
| authMethod: z$1.literal(SnowflakeAuthMethods.Password).optional(), | ||
| username: z$1.string(), | ||
| password: z$1.string() | ||
| }), | ||
| commonSnowflakeMetadataSchema.extend({ | ||
| authMethod: z.literal(SnowflakeAuthMethods.Okta), | ||
| clientId: z.string(), | ||
| clientSecret: z.string(), | ||
| oktaSubdomain: z.string(), | ||
| identityProvider: z.string(), | ||
| authorizationServer: z.string() | ||
| authMethod: z$1.literal(SnowflakeAuthMethods.Okta), | ||
| clientId: z$1.string(), | ||
| clientSecret: z$1.string(), | ||
| oktaSubdomain: z$1.string(), | ||
| identityProvider: z$1.string(), | ||
| authorizationServer: z$1.string() | ||
| }), | ||
| commonSnowflakeMetadataSchema.extend({ | ||
| authMethod: z.literal(SnowflakeAuthMethods.NativeSnowflake), | ||
| clientId: z.string(), | ||
| clientSecret: z.string() | ||
| authMethod: z$1.literal(SnowflakeAuthMethods.NativeSnowflake), | ||
| clientId: z$1.string(), | ||
| clientSecret: z$1.string() | ||
| }), | ||
| commonSnowflakeMetadataSchema.extend({ | ||
| authMethod: z.literal(SnowflakeAuthMethods.AzureAd), | ||
| clientId: z.string(), | ||
| clientSecret: z.string(), | ||
| resource: z.string(), | ||
| tenant: z.string() | ||
| authMethod: z$1.literal(SnowflakeAuthMethods.AzureAd), | ||
| clientId: z$1.string(), | ||
| clientSecret: z$1.string(), | ||
| resource: z$1.string(), | ||
| tenant: z$1.string() | ||
| }), | ||
| commonSnowflakeMetadataSchema.extend({ authMethod: z.literal(SnowflakeAuthMethods.KeyPair) }), | ||
| commonSnowflakeMetadataSchema.extend({ authMethod: z$1.literal(SnowflakeAuthMethods.KeyPair) }), | ||
| commonSnowflakeMetadataSchema.extend({ | ||
| authMethod: z.literal(SnowflakeAuthMethods.ServiceAccountKeyPair), | ||
| username: z.string(), | ||
| privateKey: z.string(), | ||
| privateKeyPassphrase: z.string().optional() | ||
| authMethod: z$1.literal(SnowflakeAuthMethods.ServiceAccountKeyPair), | ||
| username: z$1.string(), | ||
| privateKey: z$1.string(), | ||
| privateKeyPassphrase: z$1.string().optional() | ||
| }) | ||
| ])); | ||
| const spannerMetadataSchema = z.object({ | ||
| service_account: z.string(), | ||
| dataBoostEnabled: z.boolean(), | ||
| instance: z.string(), | ||
| database: z.string() | ||
| const spannerMetadataSchema = z$1.object({ | ||
| service_account: z$1.string(), | ||
| dataBoostEnabled: z$1.boolean(), | ||
| instance: z$1.string(), | ||
| database: z$1.string() | ||
| }); | ||
| const trinoBaseMetadataSchema = z.object({ | ||
| host: z.string(), | ||
| port: z.string().optional(), | ||
| database: z.string(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| const trinoBaseMetadataSchema = z$1.object({ | ||
| host: z$1.string(), | ||
| port: z$1.string().optional(), | ||
| database: z$1.string(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const trinoUsernamePasswordMetadataSchema = trinoBaseMetadataSchema.extend({ | ||
| authMethod: z.literal(TrinoAuthMethods.Password).nullish(), | ||
| user: z.string(), | ||
| password: z.string() | ||
| authMethod: z$1.literal(TrinoAuthMethods.Password).nullish(), | ||
| user: z$1.string(), | ||
| password: z$1.string() | ||
| }); | ||
| const trinoOAuthMetadataSchema = trinoBaseMetadataSchema.extend({ | ||
| authMethod: z.literal(TrinoAuthMethods.Oauth), | ||
| clientId: z.string(), | ||
| clientSecret: z.string(), | ||
| authUrl: z.string().url(), | ||
| tokenUrl: z.string().url() | ||
| authMethod: z$1.literal(TrinoAuthMethods.Oauth), | ||
| clientId: z$1.string(), | ||
| clientSecret: z$1.string(), | ||
| authUrl: z$1.string().url(), | ||
| tokenUrl: z$1.string().url() | ||
| }); | ||
| const trinoMetadataSchema = z.discriminatedUnion("authMethod", [trinoUsernamePasswordMetadataSchema, trinoOAuthMetadataSchema]); | ||
| const commonDatabaseSchema = z.object({ | ||
| host: z.string(), | ||
| user: z.string(), | ||
| password: z.string(), | ||
| database: z.string(), | ||
| sshEnabled: z.boolean().optional(), | ||
| sshHost: z.string().optional(), | ||
| sshPort: z.string().optional(), | ||
| sshUser: z.string().optional() | ||
| const trinoMetadataSchema = z$1.discriminatedUnion("authMethod", [trinoUsernamePasswordMetadataSchema, trinoOAuthMetadataSchema]); | ||
| const commonDatabaseSchema = z$1.object({ | ||
| host: z$1.string(), | ||
| user: z$1.string(), | ||
| password: z$1.string(), | ||
| database: z$1.string(), | ||
| sshEnabled: z$1.boolean().optional(), | ||
| sshHost: z$1.string().optional(), | ||
| sshPort: z$1.string().optional(), | ||
| sshUser: z$1.string().optional() | ||
| }); | ||
| const alloydbMetadataSchema = commonDatabaseSchema.extend({ | ||
| port: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| port: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const mariadbMetadataSchema = commonDatabaseSchema.extend({ | ||
| port: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| port: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const mindsdbMetadataSchema = commonDatabaseSchema.extend({ | ||
| port: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| port: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const mysqlMetadataSchema = commonDatabaseSchema.extend({ | ||
| port: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| port: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const pgsqlMetadataSchema = commonDatabaseSchema.extend({ | ||
| port: z.string().optional(), | ||
| sslEnabled: z.boolean().optional(), | ||
| caCertificateName: z.string().optional(), | ||
| caCertificateText: z.string().optional() | ||
| port: z$1.string().optional(), | ||
| sslEnabled: z$1.boolean().optional(), | ||
| caCertificateName: z$1.string().optional(), | ||
| caCertificateText: z$1.string().optional() | ||
| }); | ||
| const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: z.string() }); | ||
| const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: z.string() }); | ||
| const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: z$1.string() }); | ||
| const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: z$1.string() }); | ||
| const databaseMetadataSchemasByType = { | ||
@@ -280,2 +283,3 @@ "alloydb": alloydbMetadataSchema, | ||
| "clickhouse": clickhouseMetadataSchema, | ||
| "cloud-sql": cloudSqlMetadataSchema, | ||
| "databricks": databricksMetadataSchema, | ||
@@ -299,76 +303,81 @@ "dremio": dremioMetadataSchema, | ||
| //#region src/database-integration-config.ts | ||
| const commonIntegrationConfig = z$1.object({ | ||
| id: z$1.string(), | ||
| name: z$1.string() | ||
| const commonIntegrationConfig = z.object({ | ||
| id: z.string(), | ||
| name: z.string() | ||
| }); | ||
| const databaseIntegrationConfigSchema = z$1.discriminatedUnion("type", [ | ||
| const databaseIntegrationConfigSchema = z.discriminatedUnion("type", [ | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("alloydb"), | ||
| type: z.literal("alloydb"), | ||
| metadata: databaseMetadataSchemasByType.alloydb, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("athena"), | ||
| type: z.literal("athena"), | ||
| metadata: databaseMetadataSchemasByType.athena, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("big-query"), | ||
| type: z.literal("big-query"), | ||
| metadata: databaseMetadataSchemasByType["big-query"], | ||
| federated_auth_method: z$1.enum(["service-account", "google-oauth"]).optional().nullable() | ||
| federated_auth_method: z.enum(["service-account", "google-oauth"]).optional().nullable() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("clickhouse"), | ||
| type: z.literal("clickhouse"), | ||
| metadata: databaseMetadataSchemasByType.clickhouse, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("databricks"), | ||
| type: z.literal("cloud-sql"), | ||
| metadata: databaseMetadataSchemasByType["cloud-sql"], | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z.literal("databricks"), | ||
| metadata: databaseMetadataSchemasByType.databricks, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("dremio"), | ||
| type: z.literal("dremio"), | ||
| metadata: databaseMetadataSchemasByType.dremio, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("mariadb"), | ||
| type: z.literal("mariadb"), | ||
| metadata: databaseMetadataSchemasByType.mariadb, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("materialize"), | ||
| type: z.literal("materialize"), | ||
| metadata: databaseMetadataSchemasByType.materialize, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("mindsdb"), | ||
| type: z.literal("mindsdb"), | ||
| metadata: databaseMetadataSchemasByType.mindsdb, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("mongodb"), | ||
| type: z.literal("mongodb"), | ||
| metadata: databaseMetadataSchemasByType.mongodb, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("mysql"), | ||
| type: z.literal("mysql"), | ||
| metadata: databaseMetadataSchemasByType.mysql, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("pandas-dataframe"), | ||
| type: z.literal("pandas-dataframe"), | ||
| metadata: databaseMetadataSchemasByType["pandas-dataframe"], | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("pgsql"), | ||
| type: z.literal("pgsql"), | ||
| metadata: databaseMetadataSchemasByType.pgsql, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("redshift"), | ||
| type: z.literal("redshift"), | ||
| metadata: databaseMetadataSchemasByType.redshift, | ||
| federated_auth_method: z$1.enum([ | ||
| federated_auth_method: z.enum([ | ||
| "username-and-password", | ||
@@ -380,5 +389,5 @@ "iam-role", | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("snowflake"), | ||
| type: z.literal("snowflake"), | ||
| metadata: databaseMetadataSchemasByType.snowflake, | ||
| federated_auth_method: z$1.enum([ | ||
| federated_auth_method: z.enum([ | ||
| "okta", | ||
@@ -393,15 +402,15 @@ "snowflake", | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("spanner"), | ||
| type: z.literal("spanner"), | ||
| metadata: databaseMetadataSchemasByType.spanner, | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("sql-server"), | ||
| type: z.literal("sql-server"), | ||
| metadata: databaseMetadataSchemasByType["sql-server"], | ||
| federated_auth_method: z$1.null().optional() | ||
| federated_auth_method: z.null().optional() | ||
| }), | ||
| commonIntegrationConfig.extend({ | ||
| type: z$1.literal("trino"), | ||
| type: z.literal("trino"), | ||
| metadata: databaseMetadataSchemasByType.trino, | ||
| federated_auth_method: z$1.enum(["trino-oauth", "password"]).optional().nullable() | ||
| federated_auth_method: z.enum(["trino-oauth", "password"]).optional().nullable() | ||
| }) | ||
@@ -553,2 +562,3 @@ ]); | ||
| case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata); | ||
| case "cloud-sql": return null; | ||
| case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata); | ||
@@ -848,2 +858,3 @@ case "dremio": return getDremioSqlAlchemyInput(integration.metadata); | ||
| "clickhouse", | ||
| "cloud-sql", | ||
| "databricks", | ||
@@ -887,2 +898,196 @@ "dremio", | ||
| //#endregion | ||
| //#region src/loading/api-error.ts | ||
| /** | ||
| * Error thrown when a Deepnote API request fails. | ||
| */ | ||
| var ApiError = class extends Error { | ||
| statusCode; | ||
| constructor(statusCode, message) { | ||
| super(message); | ||
| this.name = "ApiError"; | ||
| this.statusCode = statusCode; | ||
| } | ||
| }; | ||
| //#endregion | ||
| //#region src/loading/constants.ts | ||
| /** Built-in integrations that don't require external configuration. */ | ||
| const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]); | ||
| /** Default `.env` file name for storing integration secrets. */ | ||
| const DEFAULT_ENV_FILE = ".env"; | ||
| /** Default integrations file name. */ | ||
| const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml"; | ||
| /** Default Deepnote API base URL. */ | ||
| const DEFAULT_API_URL = "https://api.deepnote.com"; | ||
| //#endregion | ||
| //#region src/loading/env-var-refs.ts | ||
| const ENV_VAR_REF_PREFIX = "env:"; | ||
| function generateEnvVarName(integrationId, fieldPath) { | ||
| return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`; | ||
| } | ||
| function parseEnvVarRef(value) { | ||
| if (typeof value !== "string") return null; | ||
| if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null; | ||
| const varName = value.slice(4); | ||
| if (!varName) return null; | ||
| return { | ||
| prefix: "env", | ||
| varName | ||
| }; | ||
| } | ||
| function createEnvVarRef(varName) { | ||
| return `${ENV_VAR_REF_PREFIX}${varName}`; | ||
| } | ||
| function isEnvVarRef(value) { | ||
| return typeof value === "string" && parseEnvVarRef(value) !== null; | ||
| } | ||
| function extractEnvVarName(value) { | ||
| if (typeof value !== "string") return null; | ||
| return parseEnvVarRef(value)?.varName ?? null; | ||
| } | ||
| /** | ||
| * Error thrown when an environment variable reference cannot be resolved. | ||
| */ | ||
| var EnvVarResolutionError = class extends Error { | ||
| varName; | ||
| path; | ||
| constructor(varName, path) { | ||
| const pathInfo = path ? ` at "${path}"` : ""; | ||
| super(`Environment variable "${varName}" is not defined${pathInfo}`); | ||
| this.name = "EnvVarResolutionError"; | ||
| this.varName = varName; | ||
| this.path = path; | ||
| } | ||
| }; | ||
| /** | ||
| * Recursively resolves environment variable references in an object using an explicit variable map. | ||
| * Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map. | ||
| * | ||
| * @param obj - The object to process (can be any value: primitive, array, or object) | ||
| * @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store) | ||
| * @param currentPath - Internal parameter for tracking the current path in the object (for error messages) | ||
| * @returns A new object with all env var references resolved | ||
| * @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved | ||
| */ | ||
| function resolveEnvVarRefsFromMap(obj, vars, currentPath) { | ||
| if (obj === null || obj === void 0) return obj; | ||
| if (typeof obj === "string") { | ||
| const parsed = parseEnvVarRef(obj); | ||
| if (parsed) { | ||
| const envValue = vars[parsed.varName]; | ||
| if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath); | ||
| return envValue; | ||
| } | ||
| return obj; | ||
| } | ||
| if (Array.isArray(obj)) return obj.map((item, index) => { | ||
| return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`); | ||
| }); | ||
| if (typeof obj === "object") { | ||
| const result = {}; | ||
| for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key); | ||
| return result; | ||
| } | ||
| return obj; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/fetch-integrations.ts | ||
| /** | ||
| * Schema for a single integration from the API. | ||
| */ | ||
| const apiIntegrationSchema = z$1.object({ | ||
| id: z$1.string(), | ||
| name: z$1.string(), | ||
| type: z$1.string(), | ||
| metadata: z$1.unknown(), | ||
| is_public: z$1.boolean(), | ||
| created_at: z$1.string(), | ||
| updated_at: z$1.string(), | ||
| federated_auth_method: z$1.string().nullable() | ||
| }).passthrough(); | ||
| /** | ||
| * Schema for the full API response. | ||
| */ | ||
| const apiResponseSchema = z$1.object({ integrations: z$1.array(apiIntegrationSchema) }).passthrough(); | ||
| /** | ||
| * Fetch integrations from the Deepnote API. | ||
| * | ||
| * Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike. | ||
| * | ||
| * @param baseUrl - The base URL of the Deepnote API | ||
| * @param token - The authentication token | ||
| * @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned. | ||
| * @returns Array of integrations from the API | ||
| * @throws ApiError if the request fails | ||
| */ | ||
| async function fetchIntegrations(baseUrl, token, integrationIds) { | ||
| const endpoint = new URL(`${baseUrl}/v2/integrations`); | ||
| endpoint.searchParams.set("includeMetadata", "true"); | ||
| if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(",")); | ||
| const url = endpoint.toString(); | ||
| const response = await fetch(url, { | ||
| method: "GET", | ||
| headers: { | ||
| Authorization: `Bearer ${token}`, | ||
| "Content-Type": "application/json" | ||
| } | ||
| }); | ||
| if (!response.ok) { | ||
| if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token."); | ||
| if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations."); | ||
| throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`); | ||
| } | ||
| const json = await response.json(); | ||
| const parseResult = apiResponseSchema.safeParse(json); | ||
| if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`); | ||
| return parseResult.data.integrations; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/integrations-document.ts | ||
| /** | ||
| * Parse integrations YAML content into a mutable `yaml` Document that preserves | ||
| * comments and formatting. Returns `null` for empty content. | ||
| * | ||
| * This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`. | ||
| */ | ||
| function parseIntegrationsDocument(content) { | ||
| if (!content.trim()) return null; | ||
| return parseDocument(content, { | ||
| strict: true, | ||
| version: "1.2" | ||
| }); | ||
| } | ||
| /** | ||
| * Serialize an integrations Document back to YAML text, preserving comments and | ||
| * formatting and avoiding line wrapping. | ||
| */ | ||
| function serializeIntegrationsDocument(doc) { | ||
| return doc.toString({ lineWidth: 0 }); | ||
| } | ||
| //#endregion | ||
| //#region src/loading/integrations-file-schema.ts | ||
| /** | ||
| * Schema for the integrations YAML file structure. | ||
| * Uses loose validation to accept any array of objects, | ||
| * allowing per-entry validation to report specific issues. | ||
| * | ||
| * Example file format: | ||
| * ```yaml | ||
| * integrations: | ||
| * - id: my-postgres | ||
| * name: My PostgreSQL | ||
| * type: pgsql | ||
| * metadata: | ||
| * host: localhost | ||
| * ... | ||
| * ``` | ||
| */ | ||
| const baseIntegrationsFileSchema = z$1.object({ integrations: z$1.array(z$1.record(z$1.unknown())).optional().default([]) }); | ||
| const integrationsFileSchema = z$1.object({ integrations: z$1.array(databaseIntegrationConfigSchema) }); | ||
| //#endregion | ||
| //#region src/secret-field-paths.ts | ||
@@ -909,2 +1114,3 @@ /** | ||
| spanner: ["service_account"], | ||
| "cloud-sql": ["service_account"], | ||
| mongodb: [ | ||
@@ -946,2 +1152,337 @@ "password", | ||
| //#endregion | ||
| export { AwsAuthMethods, BigQueryAuthMethods, BigQueryServiceAccountParseError, DatabaseAuthMethods, SnowflakeAuthMethods, SpannerServiceAccountParseError, TrinoAuthMethods, databaseIntegrationConfigSchema, databaseIntegrationTypes, databaseIntegrationTypesWithSslSupport, databaseMetadataSchemasByType, federatedAuthMethods, getEnvironmentVariablesForIntegrations, getSecretFieldPaths, getSnowflakeFederatedAuthSqlAlchemyInput, getSqlAlchemyInput, isDatabaseIntegrationType, isDatabaseIntegrationTypeWithSslSupport, isFederatedAuthMetadata, isFederatedAuthMethod, isSqlIntegrationType, sqlIntegrationTypes }; | ||
| //#region src/loading/merge-integrations.ts | ||
| /** | ||
| * JSON schema URL for the integrations file. | ||
| * TODO - change to main branch when the PR is merged into main | ||
| */ | ||
| const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json"; | ||
| /** | ||
| * Schema comment for the YAML file. | ||
| */ | ||
| const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`; | ||
| /** | ||
| * Error thrown when the integrations property has an invalid type. | ||
| */ | ||
| var InvalidIntegrationsTypeError = class extends Error { | ||
| constructor() { | ||
| super(`Invalid 'integrations' property: expected a list`); | ||
| this.name = "InvalidIntegrationsTypeError"; | ||
| } | ||
| }; | ||
| /** | ||
| * Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist. | ||
| * If the integrations property is missing or null, creates an empty array and returns a reference to it. | ||
| * Throws InvalidIntegrationsTypeError if integrations exists but is not an array. | ||
| */ | ||
| function getOrCreateIntegrationsFromDocument(doc) { | ||
| const integrations = doc.get("integrations"); | ||
| if (integrations == null) { | ||
| const emptySeq = doc.createNode([]); | ||
| emptySeq.flow = false; | ||
| doc.set("integrations", emptySeq); | ||
| return emptySeq; | ||
| } | ||
| if (!isSeq(integrations)) throw new InvalidIntegrationsTypeError(); | ||
| return integrations; | ||
| } | ||
| function getOrCreateIntegrationMetadata(doc, integrationMap) { | ||
| const existingMetadata = integrationMap.get("metadata", true); | ||
| if (isMap(existingMetadata)) return existingMetadata; | ||
| const metadata = doc.createNode({}); | ||
| integrationMap.set("metadata", metadata); | ||
| return metadata; | ||
| } | ||
| function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) { | ||
| const secrets = {}; | ||
| for (const [key, value] of Object.entries(integrationMetadata)) { | ||
| if (value === void 0) continue; | ||
| if (value === null) { | ||
| metadataMap.set(key, null); | ||
| continue; | ||
| } | ||
| if (!secretPaths.includes(key)) { | ||
| metadataMap.set(key, value); | ||
| continue; | ||
| } | ||
| const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key); | ||
| secrets[envVarName] = String(value); | ||
| metadataMap.set(key, createEnvVarRef(envVarName)); | ||
| } | ||
| return secrets; | ||
| } | ||
| /** | ||
| * Update an existing integration entry in the document. | ||
| * Preserves any comments or extra fields while updating known fields. | ||
| * Extracts secrets and replaces them with env var references, preserving custom env var names. | ||
| * | ||
| * @returns Record of extracted secrets (envVarName -> secretValue) | ||
| */ | ||
| function updateIntegrationInDocument(doc, integrationMap, integration) { | ||
| const { id, name, type, metadata, federated_auth_method,...rest } = integration; | ||
| integrationMap.set("id", id); | ||
| integrationMap.set("name", name); | ||
| integrationMap.set("type", type); | ||
| const secretPaths = getSecretFieldPaths(type); | ||
| const secrets = updateIntegrationMetadataMap({ | ||
| metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap), | ||
| integrationId: id, | ||
| integrationMetadata: metadata, | ||
| secretPaths | ||
| }); | ||
| if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method); | ||
| else integrationMap.delete("federated_auth_method"); | ||
| return secrets; | ||
| } | ||
| /** | ||
| * Add a new integration to the sequence. | ||
| * Extracts secrets and replaces them with env var references. | ||
| * | ||
| * @returns Record of extracted secrets (envVarName -> secretValue) | ||
| */ | ||
| function addIntegrationToSeq(doc, integrationsSeq, integration) { | ||
| const secretPaths = getSecretFieldPaths(integration.type); | ||
| const { metadata,...restIntegration } = integration; | ||
| const integrationMap = doc.createNode(restIntegration); | ||
| const metadataMap = doc.createNode({}); | ||
| const secrets = updateIntegrationMetadataMap({ | ||
| metadataMap, | ||
| integrationId: integration.id, | ||
| integrationMetadata: metadata, | ||
| secretPaths | ||
| }); | ||
| integrationMap.setIn(["metadata"], metadataMap); | ||
| integrationsSeq.add(integrationMap); | ||
| return secrets; | ||
| } | ||
| /** | ||
| * Create a new Document with the schema comment. | ||
| */ | ||
| function createNewDocument() { | ||
| const doc = parseDocument("integrations: []\n", { version: "1.2" }); | ||
| doc.commentBefore = SCHEMA_COMMENT; | ||
| const integrations = doc.get("integrations", true); | ||
| if (isSeq(integrations)) integrations.flow = false; | ||
| return doc; | ||
| } | ||
| /** | ||
| * Merge processed integrations into the document. | ||
| * Updates existing entries in-place (preserving comments) and adds new ones. | ||
| * Extracts secrets and replaces them with env var references during the merge. | ||
| * Preserves custom environment variable names from existing entries. | ||
| * | ||
| * @returns Object containing existing count and extracted secrets | ||
| */ | ||
| function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) { | ||
| const existingCount = integrationsSeq.items.length; | ||
| let updatedCount = 0; | ||
| let newCount = 0; | ||
| const secrets = {}; | ||
| for (const databaseIntegration of processedIntegrations) { | ||
| const existingIntegration = integrationsSeq.items.find((item) => { | ||
| if (!isMap(item)) return false; | ||
| return item.get("id") === databaseIntegration.id; | ||
| }); | ||
| if (existingIntegration != null) { | ||
| const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration); | ||
| Object.assign(secrets, extractedSecrets); | ||
| updatedCount++; | ||
| } else { | ||
| const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration); | ||
| Object.assign(secrets, extractedSecrets); | ||
| newCount++; | ||
| } | ||
| } | ||
| return { | ||
| secrets, | ||
| stats: { | ||
| existingCount, | ||
| newCount, | ||
| updatedCount | ||
| } | ||
| }; | ||
| } | ||
| /** | ||
| * Error thrown when an API integration fails schema validation during conversion. | ||
| */ | ||
| var InvalidIntegrationError = class extends Error { | ||
| integrationId; | ||
| constructor(integrationId, message) { | ||
| super(message); | ||
| this.name = "InvalidIntegrationError"; | ||
| this.integrationId = integrationId; | ||
| } | ||
| }; | ||
| /** | ||
| * Convert API integrations to DatabaseIntegrationConfig. | ||
| * Validates each integration against the schema and collects errors for invalid ones. | ||
| * | ||
| * Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching). | ||
| * | ||
| * @param apiIntegrations - Integrations fetched from the API | ||
| * @returns Object containing validated integrations and any validation errors | ||
| */ | ||
| function convertApiIntegrations(apiIntegrations) { | ||
| const integrations = []; | ||
| const errors = []; | ||
| for (const apiIntegration of apiIntegrations) { | ||
| const config = databaseIntegrationConfigSchema.safeParse(apiIntegration); | ||
| if (!config.success) { | ||
| errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API.")); | ||
| continue; | ||
| } | ||
| integrations.push(config.data); | ||
| } | ||
| return { | ||
| integrations, | ||
| errors | ||
| }; | ||
| } | ||
| /** | ||
| * Merge API integrations into an existing document (or create a new one). | ||
| * Extracts secrets and replaces them with env var references during the merge. | ||
| * Preserves custom environment variable names from existing entries. | ||
| * Invalid or unsupported integrations are silently skipped. | ||
| * | ||
| * @param doc - The YAML document to merge into | ||
| * @param apiIntegrations - Integrations fetched from the API | ||
| * @returns The extracted secrets and merge statistics | ||
| */ | ||
| function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) { | ||
| const integrationsSeq = getOrCreateIntegrationsFromDocument(doc); | ||
| const skipped = []; | ||
| return { | ||
| ...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => { | ||
| const config = databaseIntegrationConfigSchema.safeParse(apiIntegration); | ||
| if (!config.success) { | ||
| skipped.push({ | ||
| id: apiIntegration.id, | ||
| name: apiIntegration.name, | ||
| type: apiIntegration.type, | ||
| issues: config.error.issues | ||
| }); | ||
| return acc; | ||
| } | ||
| acc.push(config.data); | ||
| return acc; | ||
| }, [])), | ||
| skipped | ||
| }; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/merge-into-yaml.ts | ||
| /** | ||
| * Merge API integrations into existing integrations YAML content (or create a new | ||
| * document), returning the serialized YAML, the extracted secrets, and merge stats. | ||
| * | ||
| * This is a string-in / string-out convenience that mirrors what `deepnote | ||
| * integrations pull` does to the YAML file — usable without touching the `yaml` | ||
| * Document API. Callers persist `content` and `secrets` however they like (e.g. | ||
| * the CLI writes files; the VS Code extension can write via `workspace.fs`). | ||
| * | ||
| * @param existingContent - Current YAML content, or `null` if the file doesn't exist yet | ||
| * @param apiIntegrations - Integrations fetched from the API | ||
| */ | ||
| function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) { | ||
| const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument(); | ||
| const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations); | ||
| if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT; | ||
| return { | ||
| content: serializeIntegrationsDocument(doc), | ||
| secrets, | ||
| stats, | ||
| skipped | ||
| }; | ||
| } | ||
| //#endregion | ||
| //#region src/loading/parse-integrations.ts | ||
| /** | ||
| * Format Zod validation errors into ValidationIssue format. | ||
| */ | ||
| function formatZodIssues(issues, pathPrefix) { | ||
| return issues.map((issue) => ({ | ||
| path: [pathPrefix, ...issue.path].filter(Boolean).join("."), | ||
| message: issue.message, | ||
| code: issue.code | ||
| })); | ||
| } | ||
| /** | ||
| * Parse integrations YAML content gracefully. | ||
| * Returns all valid integrations and collects validation issues for invalid ones. | ||
| * | ||
| * This is the content-accepting, environment-agnostic core (no filesystem or | ||
| * `process.env` access) — callers supply the YAML text and the variable map used | ||
| * to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this | ||
| * to read from disk and default the map to `process.env`. | ||
| * | ||
| * @returns Object containing valid integrations and any validation issues | ||
| */ | ||
| function parseIntegrations({ yaml, env = {} }) { | ||
| const emptyIntegrations = []; | ||
| const issues = []; | ||
| let parsed; | ||
| try { | ||
| parsed = parseYaml(yaml); | ||
| } catch (error) { | ||
| const message = error instanceof Error ? error.message : String(error); | ||
| issues.push({ | ||
| path: "", | ||
| message: `Invalid YAML in integrations file: ${message}`, | ||
| code: "yaml_parse_error" | ||
| }); | ||
| return { | ||
| integrations: emptyIntegrations, | ||
| issues | ||
| }; | ||
| } | ||
| if (parsed === null || parsed === void 0) return { | ||
| integrations: emptyIntegrations, | ||
| issues | ||
| }; | ||
| const fileResult = baseIntegrationsFileSchema.safeParse(parsed); | ||
| if (!fileResult.success) { | ||
| issues.push(...formatZodIssues(fileResult.error.issues, "")); | ||
| return { | ||
| integrations: emptyIntegrations, | ||
| issues | ||
| }; | ||
| } | ||
| const entries = fileResult.data.integrations; | ||
| const integrations = []; | ||
| for (let i = 0; i < entries.length; i++) { | ||
| const entry = entries[i]; | ||
| const pathPrefix = `integrations[${i}]`; | ||
| const entryId = z$1.string().safeParse(entry.id).data; | ||
| const integrationLabel = z$1.string().safeParse(entry.name).data || entryId; | ||
| let resolvedEntry; | ||
| try { | ||
| resolvedEntry = resolveEnvVarRefsFromMap(entry, env); | ||
| } catch (error) { | ||
| if (error instanceof EnvVarResolutionError) { | ||
| const context = integrationLabel ? `Integration "${integrationLabel}": ` : ""; | ||
| issues.push({ | ||
| path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix, | ||
| message: `${context}${error.message}`, | ||
| code: "env_var_not_defined" | ||
| }); | ||
| continue; | ||
| } | ||
| throw error; | ||
| } | ||
| const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry); | ||
| if (result.success) integrations.push(result.data); | ||
| else { | ||
| const formattedIssues = formatZodIssues(result.error.issues, pathPrefix); | ||
| if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`; | ||
| issues.push(...formattedIssues); | ||
| } | ||
| } | ||
| return { | ||
| integrations, | ||
| issues | ||
| }; | ||
| } | ||
| //#endregion | ||
| export { ApiError, AwsAuthMethods, BUILTIN_INTEGRATIONS, BigQueryAuthMethods, BigQueryServiceAccountParseError, DEFAULT_API_URL, DEFAULT_ENV_FILE, DEFAULT_INTEGRATIONS_FILE, DatabaseAuthMethods, ENV_VAR_REF_PREFIX, EnvVarResolutionError, InvalidIntegrationError, InvalidIntegrationsTypeError, JSON_SCHEMA_URL, SCHEMA_COMMENT, SnowflakeAuthMethods, SpannerServiceAccountParseError, TrinoAuthMethods, addIntegrationToSeq, apiIntegrationSchema, apiResponseSchema, baseIntegrationsFileSchema, convertApiIntegrations, createEnvVarRef, createNewDocument, databaseIntegrationConfigSchema, databaseIntegrationTypes, databaseIntegrationTypesWithSslSupport, databaseMetadataSchemasByType, extractEnvVarName, federatedAuthMethods, fetchIntegrations, generateEnvVarName, getEnvironmentVariablesForIntegrations, getOrCreateIntegrationMetadata, getOrCreateIntegrationsFromDocument, getSecretFieldPaths, getSnowflakeFederatedAuthSqlAlchemyInput, getSqlAlchemyInput, getSqlEnvVarName, integrationsFileSchema, isDatabaseIntegrationType, isDatabaseIntegrationTypeWithSslSupport, isEnvVarRef, isFederatedAuthMetadata, isFederatedAuthMethod, isSqlIntegrationType, mergeApiIntegrationsIntoDocument, mergeApiIntegrationsIntoYaml, mergeProcessedIntegrations, parseEnvVarRef, parseIntegrations, parseIntegrationsDocument, resolveEnvVarRefsFromMap, serializeIntegrationsDocument, sqlIntegrationTypes, updateIntegrationInDocument, updateIntegrationMetadataMap }; |
+4
-2
| { | ||
| "name": "@deepnote/database-integrations", | ||
| "version": "1.4.3", | ||
| "version": "1.5.0", | ||
| "description": "Deepnote database integration definitions", | ||
@@ -30,3 +30,5 @@ "keywords": [], | ||
| "dependencies": { | ||
| "zod": "3.25.76" | ||
| "yaml": "^2.8.3", | ||
| "zod": "3.25.76", | ||
| "@deepnote/blocks": "4.6.0" | ||
| }, | ||
@@ -33,0 +35,0 @@ "publishConfig": { |
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Network access
Supply chain riskThis module accesses the network.
AI-detected potential code anomaly
Supply chain riskAI has identified unusual behaviors that may pose a security risk.
URL strings
Supply chain riskPackage contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.
New author
Supply chain riskA new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.
URL strings
Supply chain riskPackage contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.
535785
84.39%9235
84.07%3
200%2
100%2
100%+ Added
+ Added
+ Added
+ Added
+ Added