@stackbit/utils - npm Package Compare versions

Comparing version 0.5.1-staging.1 to 0.6.0

CHANGELOG.md

@@ -6,2 +6,18 @@ # Change Log
 
+# [0.6.0](https://github.com/stackbithq/utils/compare/@stackbit/utils@0.5.0...@stackbit/utils@0.6.0) (2024-09-24)
+
+
+### Bug Fixes
+
+* [WRFL-1747][WRFL-1741][WRFL-1744] ([2df6178](https://github.com/stackbithq/utils/commit/2df61780c9c83fbb9d9ec3b34538aa9d8a6c8aa6))
+
+
+### Features
+
+* [WRFL-1734] Added notion page fetching to AI action ([1ef92cd](https://github.com/stackbithq/utils/commit/1ef92cdf734766535025db75b8bef3d7a51c3b10))
+
+
+
+
+
 # [0.5.0](https://github.com/stackbithq/utils/compare/@stackbit/utils@0.4.28...@stackbit/utils@0.5.0) (2024-09-19)
@@ -8,0 +24,0 @@

dist/ai-actions.js

@@ -9,2 +9,5 @@ "use strict";
 const axios_1 = __importDefault(require("axios"));
+const url_1 = require("url");
+const client_1 = require("@notionhq/client");
+const notion_to_md_1 = require("notion-to-md");
 function GenerateContentFromPreset({ label, mainListField, customPrompt, allowOverrideCustomPrompt, modelsConfig }) {
@@ -43,7 +46,3 @@ return {
             }
-            const sparkClient = new SparkClient({
-                apiKey: process.env.SPARK_API_KEY,
-                url: process.env.SPARK_URL,
-                logger
-            });
+            const sparkClient = new SparkClient({ logger });
             const modelConf = (modelsConfig ?? []).find((model) => model.name === options.actionModel.name);
@@ -60,9 +59,8 @@ if (modelConf) {
             });
-            const googleDoc = await fetchGoogleDoc(options, progressCallback);
-            if (googleDoc) {
-                logger.debug('got google doc');
+            const contentMarkdown = await fetchContent(options, progressCallback, logger);
+            if (!contentMarkdown || contentMarkdown.length === 0) {
+                throw new Error('Content must not be empty');
             }
             const result = await sparkClient.performContentGen({
-                contentUrl: options.inputData?._spark_ai_content,
-                contentMarkdown: googleDoc,
+                contentMarkdown,
                 knowledge: getKnowledgeForInputData(options.inputData),
@@ -99,24 +97,23 @@ preset: options.inputData?.presetData,
 class SparkClient {
-    constructor({ apiKey, url, logger }) {
-        this.apiKey = apiKey;
-        this.url = url;
+    constructor({ logger }) {
         this.logger = logger;
     }
-    async performContentGen({ contentUrl, contentMarkdown, knowledge, preset, modelName, sectionsField, customPrompt, engineName, modelsByName, progressCallback }) {
+    async performContentGen({ contentMarkdown, knowledge, preset, modelName, sectionsField, customPrompt, engineName, modelsByName, progressCallback }) {
         const presetData = JSON.stringify(preset);
         const modelsByNameData = JSON.stringify(modelsByName);
         this.logger.debug('initialize content-gen workload');
-        if (contentMarkdown) {
-            contentUrl = undefined;
-        }
+        const baseUrl = process.env.SPARK_URL ?? 'https://api-create.services.netlify.com/spark';
         const initializedWorkload = await (0, axios_1.default)({
-            url: `${this.url}/api/v1/workload/content-gen`,
+            url: `${baseUrl}/api/v1/workload/content-gen`,
             method: 'post',
             headers: {
                 'Content-Type': 'application/json',
-                'x-spark-api-key': this.apiKey
+                ...(process.env.SPARK_API_KEY
+                    ? {
+                        'x-spark-api-key': process.env.SPARK_API_KEY
+                    }
+                    : null)
             },
             data: {
                 inputs: {
-                    url: contentUrl,
                     model: modelName,
@@ -132,9 +129,5 @@ engine: engineName ?? 'openai',
                     },
-                    ...(contentMarkdown
-                        ? {
-                            sourceData: {
-                                sourceContent: hashCode(contentMarkdown)
-                            }
-                        }
-                        : {})
+                    sourceData: {
+                        sourceContent: hashCode(contentMarkdown)
+                    }
                 },
@@ -170,12 +163,8 @@ knowledge
             }),
-            ...(contentMarkdown
-                ? [
-                    (0, axios_1.default)({
-                        url: initializedWorkloadResult.requiredUploads.sourceData.sourceContent,
-                        method: 'post',
-                        headers: { 'Content-Type': 'application/json' },
-                        data: contentMarkdown
-                    })
-                ]
-                : [])
+            (0, axios_1.default)({
+                url: initializedWorkloadResult.requiredUploads.sourceData.sourceContent,
+                method: 'post',
+                headers: { 'Content-Type': 'application/json' },
+                data: contentMarkdown
+            })
         ]);
@@ -195,2 +184,7 @@ // now that all files are uploaded, kick off the workload
         });
+        progressCallback({
+            percent: 90,
+            categoryMessage: 'Transforming content',
+            progressMessage: 'Content transformation completed'
+        });
         // fetch the result
@@ -236,8 +230,10 @@ this.logger.debug('workload finished, fetching workload result');
                     }
-                    progressCallback?.(sparkEvent);
                     const percentage = sparkEvent.latest.percentage;
-                    if (typeof percentage === 'number' && percentage === 100) {
+                    if (typeof percentage === 'number' && percentage >= 100) {
                         sse.close();
                         resolve();
                     }
+                    else {
+                        progressCallback?.(sparkEvent);
+                    }
                 }
@@ -258,3 +254,3 @@ catch (error) {
     const adjustPercentage = (percentage) => {
-        return 10 + Math.floor((percentage ?? 0) * 0.8);
+        return 10 + Math.round((percentage ?? 0) * 0.8);
     };
@@ -353,5 +349,18 @@ return (localEvent, remoteSparkEvent) => {
 }
-async function fetchGoogleDoc(options, progressCallback) {
+async function fetchContent(options, progressCallback, logger) {
+    const googleDoc = await fetchGoogleDoc(options, progressCallback, logger);
+    if (googleDoc) {
+        logger.debug('Got Google doc');
+        return googleDoc;
+    }
+    const notionPage = await fetchNotionPage(options, progressCallback, logger);
+    if (notionPage) {
+        logger.debug('Got Notion page');
+        return notionPage;
+    }
+    return options.inputData?._spark_ai_content;
+}
+async function fetchGoogleDoc(options, progressCallback, logger) {
     const googleDocUrl = options.inputData?._spark_ai_googleDocUrl ?? options.inputData?._spark_ai_content;
-    const match = googleDocUrl?.match(/docs\.google\.com\/document\/d\/([^/]+)\//);
+    const match = googleDocUrl?.match(/docs\.google\.com\/document\/d\/([^/?#]+)/);
     if (!match) {
@@ -364,2 +373,3 @@ return;
     }
+    logger.debug('Found Google doc URL');
     let googleConnection;
@@ -370,3 +380,4 @@ if (options.currentUser && 'connections' in options.currentUser && Array.isArray(options.currentUser.connections)) {
     if (!googleConnection || !('accessToken' in googleConnection)) {
-        throw new Error('Please connect your google account');
+        logger.debug('User does not have Google connection');
+        throw new Error('Please connect your Google account');
     }
@@ -378,2 +389,3 @@ progressCallback({
     });
+    logger.debug('Fetching Google doc');
     // const url = `https://docs.google.com/feeds/download/documents/export/Export?id=${documentId}&exportFormat=markdown`;
@@ -390,2 +402,55 @@ const url = `https://www.googleapis.com/drive/v3/files/${googleDocId}/export?mimeType=text/markdown`;
 }
+async function fetchNotionPage(options, progressCallback, logger) {
+    // Example for notion page URLs
+    // https://www.notion.so/{COMPANY_SLUG}/{PAGE_SLUG}-{PAGE_ID}
+    // https://www.notion.so/{PAGE_ID}
+    // https://notion.so/{PAGE_SLUG}-{PAGE_ID}
+    const notionUrl = options.inputData?._spark_ai_notionUrl ?? options.inputData?._spark_ai_content;
+    if (!/notion\.so\//.test(notionUrl)) {
+        return undefined;
+    }
+    let urlObject;
+    try {
+        urlObject = new url_1.URL(notionUrl);
+    }
+    catch (error) {
+        return undefined;
+    }
+    if (!['www.notion.so', 'notion.so'].includes(urlObject.hostname)) {
+        return undefined;
+    }
+    // The notion page ID is encoded into the last section of the URL path after the last hyphen
+    // https://www.notion.so/Hello-World-12a4dc5adgh230e5ad5ad56456b457d9
+    //                                   ↳        Notion Page ID        ↲
+    const pathParts = urlObject.pathname.replace(/^\//, '').split('/');
+    const notionPageSlugAndId = pathParts[pathParts.length - 1];
+    if (!notionPageSlugAndId) {
+        return undefined;
+    }
+    const pageSlugAndIdParts = notionPageSlugAndId.split('-');
+    const notionPageId = pageSlugAndIdParts[pageSlugAndIdParts.length - 1];
+    if (!notionPageId) {
+        return undefined;
+    }
+    logger.debug(`Found Notion page URL with ID: ${notionPageId}`);
+    let notionConnection;
+    if (options.currentUser && 'connections' in options.currentUser && Array.isArray(options.currentUser.connections)) {
+        notionConnection = options.currentUser.connections.find((connection) => connection.type === 'notion');
+    }
+    if (!notionConnection || !('accessToken' in notionConnection)) {
+        logger.debug('User does not have Notion connection');
+        throw new Error('Please connect your Notion account');
+    }
+    progressCallback({
+        percent: 2,
+        categoryMessage: 'Initializing',
+        progressMessage: `Fetching from ${notionUrl}`
+    });
+    logger.debug('Fetching Notion page');
+    const notionClient = new client_1.Client({ auth: notionConnection.accessToken });
+    const n2m = new notion_to_md_1.NotionToMarkdown({ notionClient });
+    const mdBlocks = await n2m.pageToMarkdown(notionPageId);
+    const mdString = n2m.toMarkdownString(mdBlocks);
+    return mdString.parent;
+}
 exports.Actions = {
@@ -392,0 +457,0 @@ GenerateContentFromPreset

dist/package-manager.d.ts

 /// <reference types="node" />
-/// <reference types="node" />
 export interface PackageManagerDetails {
@@ -4,0 +3,0 @@ name: string;

dist/process-utils.d.ts

 /// <reference types="node" />
-/// <reference types="node" />
 import { ChildProcessWithoutNullStreams } from 'child_process';
@@ -4,0 +3,0 @@ import { Logger } from './logger-types';

jest.config.js

 module.exports = {
     testPathIgnorePatterns: ['__tests__/test-utils'],
     preset: 'ts-jest',
-    testEnvironment: 'node'
+    testEnvironment: 'node',
+    moduleNameMapper: {
+        '^axios$': 'axios/dist/node/axios.cjs'
+    }
 };

package.json

 {
   "name": "@stackbit/utils",
-  "version": "0.5.1-staging.1",
+  "version": "0.6.0",
   "description": "Stackbit utilities",
@@ -29,3 +29,4 @@ "main": "dist/index.js",
     "@iarna/toml": "^2.2.5",
-    "@stackbit/types": "2.0.8-staging.1",
+    "@notionhq/client": "^2.2.15",
+    "@stackbit/types": "2.0.7",
     "axios": "^1.7.7",
@@ -36,2 +37,3 @@ "eventsource": "^2.0.2",
     "lodash": "^4.17.21",
+    "notion-to-md": "^3.1.1",
     "strip-ansi": "^6.0.1"
@@ -45,3 +47,3 @@ },
   },
-  "gitHead": "86e254f54ce4bb449cd9800724dd4c5e8fa83df8"
+  "gitHead": "e485d15ee3e2b7ef4b6fa5cb28705783339e89ee"
 }

src/ai-actions.ts

 import EventSource from 'eventsource';
 import axios from 'axios';
+import { URL } from 'url';
+import { Client as NotionClient } from '@notionhq/client';
+import { NotionToMarkdown } from 'notion-to-md';
 import {
@@ -86,7 +89,3 @@ CustomActionModel,
 
-            const sparkClient = new SparkClient({
-                apiKey: process.env.SPARK_API_KEY!,
-                url: process.env.SPARK_URL!,
-                logger
-            });
+            const sparkClient = new SparkClient({ logger });
 
@@ -107,10 +106,10 @@ const modelConf = (modelsConfig ?? []).find((model) => model.name === options.actionModel.name);
 
-            const googleDoc = await fetchGoogleDoc(options, progressCallback);
-            if (googleDoc) {
-                logger.debug('got google doc');
+            const contentMarkdown = await fetchContent(options, progressCallback, logger);
+
+            if (!contentMarkdown || contentMarkdown.length === 0) {
+                throw new Error('Content must not be empty');
             }
 
             const result = await sparkClient.performContentGen({
-                contentUrl: options.inputData?._spark_ai_content,
-                contentMarkdown: googleDoc,
+                contentMarkdown,
                 knowledge: getKnowledgeForInputData(options.inputData),
@@ -150,9 +149,5 @@ preset: options.inputData?.presetData,
 class SparkClient {
-    private readonly apiKey: string;
-    private readonly url: string;
     private readonly logger: Logger;
 
-    constructor({ apiKey, url, logger }: { apiKey: string; url: string; logger: any }) {
-        this.apiKey = apiKey;
-        this.url = url;
+    constructor({ logger }: { logger: any }) {
         this.logger = logger;
@@ -162,3 +157,2 @@ }
     async performContentGen({
-        contentUrl,
         contentMarkdown,
@@ -174,4 +168,3 @@ knowledge,
     }: {
-        contentUrl?: string;
-        contentMarkdown?: string;
+        contentMarkdown: string;
         knowledge?: SparkKnowledge;
@@ -191,16 +184,16 @@ preset: any;
 
-        if (contentMarkdown) {
-            contentUrl = undefined;
-        }
-
+        const baseUrl = process.env.SPARK_URL ?? 'https://api-create.services.netlify.com/spark';
         const initializedWorkload = await axios({
-            url: `${this.url}/api/v1/workload/content-gen`,
+            url: `${baseUrl}/api/v1/workload/content-gen`,
             method: 'post',
             headers: {
                 'Content-Type': 'application/json',
-                'x-spark-api-key': this.apiKey
+                ...(process.env.SPARK_API_KEY
+                    ? {
+                          'x-spark-api-key': process.env.SPARK_API_KEY
+                      }
+                    : null)
             },
             data: {
                 inputs: {
-                    url: contentUrl,
                     model: modelName,
@@ -216,9 +209,5 @@ engine: engineName ?? 'openai',
                     },
-                    ...(contentMarkdown
-                        ? {
-                              sourceData: {
-                                  sourceContent: hashCode(contentMarkdown)
-                              }
-                          }
-                        : {})
+                    sourceData: {
+                        sourceContent: hashCode(contentMarkdown)
+                    }
                 },
@@ -257,12 +246,8 @@ knowledge
             }),
-            ...(contentMarkdown
-                ? [
-                      axios({
-                          url: initializedWorkloadResult.requiredUploads.sourceData.sourceContent,
-                          method: 'post',
-                          headers: { 'Content-Type': 'application/json' },
-                          data: contentMarkdown
-                      })
-                  ]
-                : [])
+            axios({
+                url: initializedWorkloadResult.requiredUploads.sourceData.sourceContent,
+                method: 'post',
+                headers: { 'Content-Type': 'application/json' },
+                data: contentMarkdown
+            })
         ]);
@@ -285,2 +270,8 @@
 
+        progressCallback({
+            percent: 90,
+            categoryMessage: 'Transforming content',
+            progressMessage: 'Content transformation completed'
+        });
+
         // fetch the result
@@ -338,7 +329,8 @@ this.logger.debug('workload finished, fetching workload result');
                     }
-                    progressCallback?.(sparkEvent);
                     const percentage = sparkEvent.latest.percentage;
-                    if (typeof percentage === 'number' && percentage === 100) {
+                    if (typeof percentage === 'number' && percentage >= 100) {
                         sse.close();
                         resolve();
+                    } else {
+                        progressCallback?.(sparkEvent);
                     }
@@ -376,3 +368,3 @@ } catch (error) {
     const adjustPercentage = (percentage?: number) => {
-        return 10 + Math.floor((percentage ?? 0) * 0.8);
+        return 10 + Math.round((percentage ?? 0) * 0.8);
     };
@@ -498,5 +490,21 @@
 
-async function fetchGoogleDoc(options: CustomActionRunCommonOptions, progressCallback: ProgressCallback): Promise<string | undefined> {
+async function fetchContent(options: CustomActionRunCommonOptions, progressCallback: ProgressCallback, logger: Logger): Promise<string> {
+    const googleDoc = await fetchGoogleDoc(options, progressCallback, logger);
+    if (googleDoc) {
+        logger.debug('Got Google doc');
+        return googleDoc;
+    }
+
+    const notionPage = await fetchNotionPage(options, progressCallback, logger);
+    if (notionPage) {
+        logger.debug('Got Notion page');
+        return notionPage;
+    }
+
+    return options.inputData?._spark_ai_content;
+}
+
+async function fetchGoogleDoc(options: CustomActionRunCommonOptions, progressCallback: ProgressCallback, logger: Logger): Promise<string | undefined> {
     const googleDocUrl = options.inputData?._spark_ai_googleDocUrl ?? options.inputData?._spark_ai_content;
-    const match = googleDocUrl?.match(/docs\.google\.com\/document\/d\/([^/]+)\//);
+    const match = googleDocUrl?.match(/docs\.google\.com\/document\/d\/([^/?#]+)/);
     if (!match) {
@@ -510,2 +518,4 @@ return;
 
+    logger.debug('Found Google doc URL');
+
     let googleConnection;
@@ -516,3 +526,4 @@ if (options.currentUser && 'connections' in options.currentUser && Array.isArray(options.currentUser.connections)) {
     if (!googleConnection || !('accessToken' in googleConnection)) {
-        throw new Error('Please connect your google account');
+        logger.debug('User does not have Google connection');
+        throw new Error('Please connect your Google account');
     }
@@ -526,2 +537,3 @@
 
+    logger.debug('Fetching Google doc');
     // const url = `https://docs.google.com/feeds/download/documents/export/Export?id=${documentId}&exportFormat=markdown`;
@@ -539,4 +551,62 @@ const url = `https://www.googleapis.com/drive/v3/files/${googleDocId}/export?mimeType=text/markdown`;
 
+async function fetchNotionPage(options: CustomActionRunCommonOptions, progressCallback: ProgressCallback, logger: Logger): Promise<string | undefined> {
+    // Example for notion page URLs
+    // https://www.notion.so/{COMPANY_SLUG}/{PAGE_SLUG}-{PAGE_ID}
+    // https://www.notion.so/{PAGE_ID}
+    // https://notion.so/{PAGE_SLUG}-{PAGE_ID}
+    const notionUrl = options.inputData?._spark_ai_notionUrl ?? options.inputData?._spark_ai_content;
+    if (!/notion\.so\//.test(notionUrl)) {
+        return undefined;
+    }
+    let urlObject: URL;
+    try {
+        urlObject = new URL(notionUrl);
+    } catch (error) {
+        return undefined;
+    }
+    if (!['www.notion.so', 'notion.so'].includes(urlObject.hostname)) {
+        return undefined;
+    }
+
+    // The notion page ID is encoded into the last section of the URL path after the last hyphen
+    // https://www.notion.so/Hello-World-12a4dc5adgh230e5ad5ad56456b457d9
+    //                                   ↳        Notion Page ID        ↲
+    const pathParts = urlObject.pathname.replace(/^\//, '').split('/');
+    const notionPageSlugAndId = pathParts[pathParts.length - 1];
+    if (!notionPageSlugAndId) {
+        return undefined;
+    }
+    const pageSlugAndIdParts = notionPageSlugAndId.split('-');
+    const notionPageId = pageSlugAndIdParts[pageSlugAndIdParts.length - 1];
+    if (!notionPageId) {
+        return undefined;
+    }
+
+    logger.debug(`Found Notion page URL with ID: ${notionPageId}`);
+
+    let notionConnection;
+    if (options.currentUser && 'connections' in options.currentUser && Array.isArray(options.currentUser.connections)) {
+        notionConnection = options.currentUser.connections.find((connection) => connection.type === 'notion');
+    }
+    if (!notionConnection || !('accessToken' in notionConnection)) {
+        logger.debug('User does not have Notion connection');
+        throw new Error('Please connect your Notion account');
+    }
+
+    progressCallback({
+        percent: 2,
+        categoryMessage: 'Initializing',
+        progressMessage: `Fetching from ${notionUrl}`
+    });
+
+    logger.debug('Fetching Notion page');
+    const notionClient = new NotionClient({ auth: notionConnection.accessToken });
+    const n2m = new NotionToMarkdown({ notionClient });
+    const mdBlocks = await n2m.pageToMarkdown(notionPageId);
+    const mdString = n2m.toMarkdownString(mdBlocks);
+    return mdString.parent;
+}
+
 export const Actions = {
     GenerateContentFromPreset
 };

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

552600

increased by1.6%

Lines of code

10307

increased by1.23%

Worsened metrics

Dependency count

10

increased by25%

Number of low supply chain risk alerts

6

increased by20%

Dependency changes

• + Added@notionhq/client@^2.2.15

• + Addednotion-to-md@^3.1.1

• + Added@notionhq/client@2.2.15(transitive)

• + Added@stackbit/types@2.0.7(transitive)

• + Added@types/node@22.9.0(transitive)

• + Added@types/node-fetch@2.6.12(transitive)

• + Addedmarkdown-table@2.0.0(transitive)

• + Addednode-fetch@2.7.0(transitive)

• + Addednotion-to-md@3.1.1(transitive)

• + Addedrepeat-string@1.6.1(transitive)

• + Addedtr46@0.0.3(transitive)

• + Addedundici-types@6.19.8(transitive)

• + Addedwebidl-conversions@3.0.1(transitive)

• + Addedwhatwg-url@5.0.0(transitive)

• - Removed@stackbit/types@2.0.8-staging.1(transitive)

• Updated@stackbit/types@2.0.7