@stackbit/utils - npm Package Compare versions

Comparing version 0.4.28 to 0.5.0

CHANGELOG.md

@@ -6,2 +6,13 @@ # Change Log
 
+# [0.5.0](https://github.com/stackbithq/utils/compare/@stackbit/utils@0.4.28...@stackbit/utils@0.5.0) (2024-09-19)
+
+
+### Features
+
+* [WRFL-1702] adding support for google doc in ai actions ([208a7c1](https://github.com/stackbithq/utils/commit/208a7c1a0e3c572f2ab70ee6f810f43c48007d86))
+
+
+
+
+
 ## [0.4.28](https://github.com/stackbithq/utils/compare/@stackbit/utils@0.4.27...@stackbit/utils@0.4.28) (2024-09-10)
@@ -8,0 +19,0 @@

dist/ai-actions.js

@@ -17,7 +17,2 @@ "use strict";
             {
-                type: 'string',
-                name: 'contentUrl',
-                label: 'Content Url'
-            },
-            {
                 type: 'slug',
@@ -58,4 +53,16 @@ name: 'slug',
             }
+            const progressCallback = createProgressCallback(options.progress);
+            progressCallback({
+                percent: 0,
+                categoryMessage: 'Initializing',
+                progressMessage: 'Initializing'
+            });
+            const googleDoc = await fetchGoogleDoc(options, progressCallback);
+            if (googleDoc) {
+                logger.debug('got google doc');
+            }
             const result = await sparkClient.performContentGen({
-                contentUrl: options.inputData?.contentUrl,
+                contentUrl: options.inputData?._spark_ai_content,
+                contentMarkdown: googleDoc,
+                knowledge: getKnowledgeForInputData(options.inputData),
                 preset: options.inputData?.presetData,
@@ -67,3 +74,3 @@ customPrompt: options.inputData?.customPrompt ?? customPrompt,
                 modelsByName,
-                progressCallback: options.progress
+                progressCallback
             });
@@ -74,7 +81,7 @@ if (options.inputData?.slug) {
             logger.info('create document');
-            options.progress(createProgressEvent({
+            progressCallback({
                 percent: 95,
                 categoryMessage: 'Saving content',
                 progressMessage: 'Saving content'
-            }));
+            });
             const { documentId } = await options.contentSourceActions.createDocumentFromObject({
@@ -98,11 +105,9 @@ modelName: options.actionModel.name,
     }
-    async performContentGen({ contentUrl, preset, modelName, sectionsField, customPrompt, engineName, modelsByName, progressCallback }) {
+    async performContentGen({ contentUrl, contentMarkdown, knowledge, preset, modelName, sectionsField, customPrompt, engineName, modelsByName, progressCallback }) {
         const presetData = JSON.stringify(preset);
         const modelsByNameData = JSON.stringify(modelsByName);
         this.logger.debug('initialize content-gen workload');
-        progressCallback?.(createProgressEvent({
-            percent: 0,
-            categoryMessage: 'Initializing',
-            progressMessage: 'Initializing'
-        }));
+        if (contentMarkdown) {
+            contentUrl = undefined;
+        }
         const initializedWorkload = await (0, axios_1.default)({
@@ -127,4 +132,12 @@ url: `${this.url}/api/v1/workload/content-gen`,
                         modelsByName: hashCode(modelsByNameData)
-                    }
-                }
+                    },
+                    ...(contentMarkdown
+                        ? {
+                            sourceData: {
+                                sourceContent: hashCode(contentMarkdown)
+                            }
+                        }
+                        : {})
+                },
+                knowledge
             }
@@ -134,3 +147,3 @@ });
         const initializedWorkloadResult = initializedWorkload.data;
-        this.logger.debug('initialized workload, uploading data', { initializedWorkloadResult });
+        this.logger.debug('initialized workload');
         if (!initializedWorkloadResult.startWorkload) {
@@ -141,7 +154,7 @@ throw new Error('workload must specify startWorkload URL');
         this.logger.debug('uploading workload createData');
-        progressCallback?.(createProgressEvent({
+        progressCallback({
             percent: 5,
             categoryMessage: 'Initializing',
             progressMessage: 'Uploading content'
-        }));
+        });
         await Promise.all([
@@ -159,10 +172,20 @@ (0, axios_1.default)({
                 data: modelsByName
-            })
+            }),
+            ...(contentMarkdown
+                ? [
+                    (0, axios_1.default)({
+                        url: initializedWorkloadResult.requiredUploads.sourceData.sourceContent,
+                        method: 'post',
+                        headers: { 'Content-Type': 'application/json' },
+                        data: contentMarkdown
+                    })
+                ]
+                : [])
         ]);
         // now that all files are uploaded, kick off the workload
-        const startWorkloadResult = await (0, axios_1.default)({
+        await (0, axios_1.default)({
             url: initializedWorkloadResult.startWorkload,
             method: 'post'
         });
-        this.logger.debug('started workload', { startWorkloadResult: startWorkloadResult.data });
+        this.logger.debug('started workload');
         // wait until the workload is done
@@ -172,18 +195,3 @@ await this.waitUntilDone({
             progressCallback: (sparkEvent) => {
-                let percent = 10;
-                if (sparkEvent.latest?.percentage) {
-                    percent = 10 + Math.floor(sparkEvent.latest.percentage * 0.8);
-                    sparkEvent.latest.percentage = percent;
-                    sparkEvent = {
-                        ...sparkEvent,
-                        latest: {
-                            ...sparkEvent.latest,
-                            percentage: percent
-                        }
-                    };
-                }
-                progressCallback?.({
-                    percent: percent,
-                    message: JSON.stringify(sparkEvent)
-                });
+                progressCallback(null, sparkEvent);
             }
@@ -197,7 +205,10 @@ });
         });
-        this.logger.debug('got workload result', { workloadResult });
+        this.logger.debug('got workload result', {
+            status: workloadResult.status,
+            statusText: workloadResult.statusText
+        });
         return workloadResult.data;
     }
     waitUntilDone({ sseUrl, progressCallback, retryCount = 0 }) {
-        this.logger.debug('subscribe to server-side-events', { retryCount, sseUrl });
+        this.logger.debug('subscribe to server-side-events', { retryCount });
         const sse = new eventsource_1.default(sseUrl);
@@ -226,3 +237,7 @@ return new Promise((resolve, reject) => {
                     const sparkEvent = JSON.parse(event.data);
-                    const percentage = sparkEvent.latest?.percentage;
+                    if (!sparkEvent.latest) {
+                        return;
+                    }
+                    progressCallback?.(sparkEvent);
+                    const percentage = sparkEvent.latest.percentage;
                     if (typeof percentage === 'number' && percentage === 100) {
@@ -232,5 +247,2 @@ sse.close();
                     }
-                    else {
-                        progressCallback?.(sparkEvent);
-                    }
                 }
@@ -245,13 +257,64 @@ catch (error) {
 }
-function createProgressEvent(options) {
+function createProgressCallback(progressCallback) {
+    const localProgressBefore = [];
+    const localProgressAfter = [];
+    let remoteProgress = [];
+    let gotRemoteEvents = false;
+    const adjustPercentage = (percentage) => {
+        return 10 + Math.floor((percentage ?? 0) * 0.8);
+    };
+    return (localEvent, remoteSparkEvent) => {
+        let sparkEvent;
+        if (remoteSparkEvent) {
+            gotRemoteEvents = true;
+            remoteProgress =
+                remoteSparkEvent.allProgress?.map((step) => {
+                    return {
+                        ...step,
+                        percentage: adjustPercentage(step.percentage)
+                    };
+                }) ?? remoteProgress;
+            if (remoteSparkEvent.latest) {
+                sparkEvent = {
+                    ...remoteSparkEvent,
+                    latest: {
+                        ...remoteSparkEvent.latest,
+                        percentage: adjustPercentage(remoteSparkEvent.latest.percentage)
+                    },
+                    allProgress: localProgressBefore.concat(remoteProgress ?? [])
+                };
+            }
+        }
+        else if (localEvent) {
+            const latest = createSparkEventProgressMessage(localEvent);
+            if (!gotRemoteEvents) {
+                localProgressBefore.push(latest);
+                sparkEvent = {
+                    latest,
+                    allProgress: localProgressBefore
+                };
+            }
+            else {
+                localProgressAfter.push(latest);
+                sparkEvent = {
+                    latest,
+                    allProgress: localProgressBefore.concat(remoteProgress, localProgressAfter)
+                };
+            }
+        }
+        if (sparkEvent?.latest) {
+            progressCallback?.({
+                percent: sparkEvent.latest?.percentage,
+                message: JSON.stringify(sparkEvent)
+            });
+        }
+    };
+}
+function createSparkEventProgressMessage(options) {
     return {
-        percent: options.percent,
-        message: JSON.stringify({
-            latest: {
-                categoryMessage: options.categoryMessage,
-                progressMessage: options.progressMessage,
-                percentage: options.percent
-            },
-            allProgress: []
-        })
+        percentage: options.percent,
+        categoryMessage: options.categoryMessage,
+        progressMessage: options.progressMessage,
+        systemFailure: options.systemFailure,
+        errorMessage: options.errorMessage
     };
@@ -268,2 +331,62 @@ }
 }
+function getKnowledgeForInputData(inputData) {
+    if (!inputData) {
+        return undefined;
+    }
+    const selected = [];
+    if (inputData._spark_ai_voiceTone) {
+        selected.push({
+            type: 'voice-and-tone',
+            id: inputData._spark_ai_voiceTone
+        });
+    }
+    if (inputData._spark_ai_targetAudience) {
+        selected.push({
+            type: 'target-audience',
+            id: inputData._spark_ai_targetAudience
+        });
+    }
+    if (selected.length === 0) {
+        return undefined;
+    }
+    return {
+        scopes: {
+            shared: 'netlify-known'
+        },
+        selected
+    };
+}
+async function fetchGoogleDoc(options, progressCallback) {
+    const googleDocUrl = options.inputData?._spark_ai_googleDocUrl ?? options.inputData?._spark_ai_content;
+    const match = googleDocUrl?.match(/docs\.google\.com\/document\/d\/([^/]+)\//);
+    if (!match) {
+        return;
+    }
+    const googleDocId = match[1];
+    if (!googleDocId) {
+        return;
+    }
+    let googleConnection;
+    if (options.currentUser && 'connections' in options.currentUser && Array.isArray(options.currentUser.connections)) {
+        googleConnection = options.currentUser.connections.find((connection) => connection.type === 'google');
+    }
+    if (!googleConnection || !('accessToken' in googleConnection)) {
+        throw new Error('Please connect your google account');
+    }
+    progressCallback({
+        percent: 2,
+        categoryMessage: 'Initializing',
+        progressMessage: `Fetching from ${googleDocUrl}`
+    });
+    // const url = `https://docs.google.com/feeds/download/documents/export/Export?id=${documentId}&exportFormat=markdown`;
+    const url = `https://www.googleapis.com/drive/v3/files/${googleDocId}/export?mimeType=text/markdown`;
+    const googleDocResult = await (0, axios_1.default)({
+        url: url,
+        method: 'get',
+        headers: {
+            Authorization: `Bearer ${googleConnection.accessToken}`
+        }
+    });
+    return googleDocResult.data;
+}
 exports.Actions = {
@@ -270,0 +393,0 @@ GenerateContentFromPreset

package.json

 {
   "name": "@stackbit/utils",
-  "version": "0.4.28",
+  "version": "0.5.0",
   "description": "Stackbit utilities",
@@ -43,3 +43,3 @@ "main": "dist/index.js",
   },
-  "gitHead": "7e6ef4b9a6f097e0a99ccb6f7dec6eaf1bbc5ee0"
+  "gitHead": "f59975a51b21abe88bf152d0cd8525ba01ae0c8b"
 }

src/ai-actions.ts

 import EventSource from 'eventsource';
 import axios from 'axios';
-import { CustomActionModel, CustomActionRunCommonOptions, CustomActionModelRunOptions, ModelMap, Logger, CustomActionInputField } from '@stackbit/types';
+import {
+    CustomActionModel,
+    CustomActionRunCommonOptions,
+    CustomActionModelRunOptions,
+    CustomActionInputField,
+    CustomActionProgressFunction,
+    ModelMap,
+    Logger
+} from '@stackbit/types';
 
@@ -51,7 +59,2 @@ export type GenerateContentFromPresetOptions = {
             {
-                type: 'string',
-                name: 'contentUrl',
-                label: 'Content Url'
-            },
-            {
                 type: 'slug',
@@ -96,4 +99,19 @@ name: 'slug',
 
+            const progressCallback = createProgressCallback(options.progress);
+
+            progressCallback({
+                percent: 0,
+                categoryMessage: 'Initializing',
+                progressMessage: 'Initializing'
+            });
+
+            const googleDoc = await fetchGoogleDoc(options, progressCallback);
+            if (googleDoc) {
+                logger.debug('got google doc');
+            }
+
             const result = await sparkClient.performContentGen({
-                contentUrl: options.inputData?.contentUrl,
+                contentUrl: options.inputData?._spark_ai_content,
+                contentMarkdown: googleDoc,
+                knowledge: getKnowledgeForInputData(options.inputData),
                 preset: options.inputData?.presetData,
@@ -105,3 +123,3 @@ customPrompt: options.inputData?.customPrompt ?? customPrompt,
                 modelsByName,
-                progressCallback: options.progress
+                progressCallback
             });
@@ -114,9 +132,7 @@
             logger.info('create document');
-            options.progress(
-                createProgressEvent({
-                    percent: 95,
-                    categoryMessage: 'Saving content',
-                    progressMessage: 'Saving content'
-                })
-            );
+            progressCallback({
+                percent: 95,
+                categoryMessage: 'Saving content',
+                progressMessage: 'Saving content'
+            });
             const { documentId } = await options.contentSourceActions.createDocumentFromObject({
@@ -148,2 +164,4 @@ modelName: options.actionModel.name,
         contentUrl,
+        contentMarkdown,
+        knowledge,
         preset,
@@ -157,3 +175,5 @@ modelName,
     }: {
-        contentUrl: string;
+        contentUrl?: string;
+        contentMarkdown?: string;
+        knowledge?: SparkKnowledge;
         preset: any;
@@ -165,3 +185,3 @@ modelName: string;
         modelsByName: Record<string, any>;
-        progressCallback?: (options: { message?: string; percent?: number }) => void;
+        progressCallback: ProgressCallback;
     }) {
@@ -172,9 +192,7 @@ const presetData = JSON.stringify(preset);
         this.logger.debug('initialize content-gen workload');
-        progressCallback?.(
-            createProgressEvent({
-                percent: 0,
-                categoryMessage: 'Initializing',
-                progressMessage: 'Initializing'
-            })
-        );
+
+        if (contentMarkdown) {
+            contentUrl = undefined;
+        }
+
         const initializedWorkload = await axios({
@@ -199,4 +217,12 @@ url: `${this.url}/api/v1/workload/content-gen`,
                         modelsByName: hashCode(modelsByNameData)
-                    }
-                }
+                    },
+                    ...(contentMarkdown
+                        ? {
+                              sourceData: {
+                                  sourceContent: hashCode(contentMarkdown)
+                              }
+                          }
+                        : {})
+                },
+                knowledge
             }
@@ -207,3 +233,3 @@ });
         const initializedWorkloadResult = initializedWorkload.data;
-        this.logger.debug('initialized workload, uploading data', { initializedWorkloadResult });
+        this.logger.debug('initialized workload');
 
@@ -216,9 +242,7 @@ if (!initializedWorkloadResult.startWorkload) {
         this.logger.debug('uploading workload createData');
-        progressCallback?.(
-            createProgressEvent({
-                percent: 5,
-                categoryMessage: 'Initializing',
-                progressMessage: 'Uploading content'
-            })
-        );
+        progressCallback({
+            percent: 5,
+            categoryMessage: 'Initializing',
+            progressMessage: 'Uploading content'
+        });
         await Promise.all([
@@ -236,11 +260,21 @@ axios({
                 data: modelsByName
-            })
+            }),
+            ...(contentMarkdown
+                ? [
+                      axios({
+                          url: initializedWorkloadResult.requiredUploads.sourceData.sourceContent,
+                          method: 'post',
+                          headers: { 'Content-Type': 'application/json' },
+                          data: contentMarkdown
+                      })
+                  ]
+                : [])
         ]);
 
         // now that all files are uploaded, kick off the workload
-        const startWorkloadResult = await axios({
+        await axios({
             url: initializedWorkloadResult.startWorkload,
             method: 'post'
         });
-        this.logger.debug('started workload', { startWorkloadResult: startWorkloadResult.data });
+        this.logger.debug('started workload');
 
@@ -251,18 +285,3 @@ // wait until the workload is done
             progressCallback: (sparkEvent: SparkEvent) => {
-                let percent = 10;
-                if (sparkEvent.latest?.percentage) {
-                    percent = 10 + Math.floor(sparkEvent.latest.percentage * 0.8);
-                    sparkEvent.latest.percentage = percent;
-                    sparkEvent = {
-                        ...sparkEvent,
-                        latest: {
-                            ...sparkEvent.latest,
-                            percentage: percent
-                        }
-                    };
-                }
-                progressCallback?.({
-                    percent: percent,
-                    message: JSON.stringify(sparkEvent)
-                });
+                progressCallback(null, sparkEvent);
             }
@@ -277,3 +296,6 @@ });
         });
-        this.logger.debug('got workload result', { workloadResult });
+        this.logger.debug('got workload result', {
+            status: workloadResult.status,
+            statusText: workloadResult.statusText
+        });
 
@@ -292,3 +314,3 @@ return workloadResult.data;
     }): Promise<void> {
-        this.logger.debug('subscribe to server-side-events', { retryCount, sseUrl });
+        this.logger.debug('subscribe to server-side-events', { retryCount });
         const sse = new EventSource(sseUrl);
@@ -319,8 +341,10 @@ return new Promise((resolve, reject) => {
                     const sparkEvent = JSON.parse(event.data) as SparkEvent;
-                    const percentage = sparkEvent.latest?.percentage;
+                    if (!sparkEvent.latest) {
+                        return;
+                    }
+                    progressCallback?.(sparkEvent);
+                    const percentage = sparkEvent.latest.percentage;
                     if (typeof percentage === 'number' && percentage === 100) {
                         sse.close();
                         resolve();
-                    } else {
-                        progressCallback?.(sparkEvent);
                     }
@@ -345,15 +369,77 @@ } catch (error) {
     progressMessage?: string;
+    systemFailure?: boolean;
+    errorMessage?: string;
 };
 
-function createProgressEvent(options: { percent: number; categoryMessage: string; progressMessage: string }): { message?: string; percent?: number } {
+type ProgressCallback = (localEvent: { percent: number; categoryMessage: string; progressMessage: string } | null, remoteSparkEvent?: SparkEvent) => void;
+
+function createProgressCallback(progressCallback: CustomActionProgressFunction): ProgressCallback {
+    const localProgressBefore: SparkEventProgressMessage[] = [];
+    const localProgressAfter: SparkEventProgressMessage[] = [];
+    let remoteProgress: SparkEventProgressMessage[] = [];
+    let gotRemoteEvents = false;
+
+    const adjustPercentage = (percentage?: number) => {
+        return 10 + Math.floor((percentage ?? 0) * 0.8);
+    };
+
+    return (localEvent: { percent: number; categoryMessage: string; progressMessage: string } | null, remoteSparkEvent?: SparkEvent) => {
+        let sparkEvent: SparkEvent | undefined;
+        if (remoteSparkEvent) {
+            gotRemoteEvents = true;
+            remoteProgress =
+                remoteSparkEvent.allProgress?.map((step) => {
+                    return {
+                        ...step,
+                        percentage: adjustPercentage(step.percentage)
+                    };
+                }) ?? remoteProgress;
+            if (remoteSparkEvent.latest) {
+                sparkEvent = {
+                    ...remoteSparkEvent,
+                    latest: {
+                        ...remoteSparkEvent.latest,
+                        percentage: adjustPercentage(remoteSparkEvent.latest.percentage)
+                    },
+                    allProgress: localProgressBefore.concat(remoteProgress ?? [])
+                };
+            }
+        } else if (localEvent) {
+            const latest = createSparkEventProgressMessage(localEvent);
+            if (!gotRemoteEvents) {
+                localProgressBefore.push(latest);
+                sparkEvent = {
+                    latest,
+                    allProgress: localProgressBefore
+                };
+            } else {
+                localProgressAfter.push(latest);
+                sparkEvent = {
+                    latest,
+                    allProgress: localProgressBefore.concat(remoteProgress, localProgressAfter)
+                };
+            }
+        }
+        if (sparkEvent?.latest) {
+            progressCallback?.({
+                percent: sparkEvent.latest?.percentage,
+                message: JSON.stringify(sparkEvent)
+            });
+        }
+    };
+}
+
+function createSparkEventProgressMessage(options: {
+    percent: number;
+    categoryMessage: string;
+    progressMessage: string;
+    systemFailure?: boolean;
+    errorMessage?: string;
+}): SparkEventProgressMessage {
     return {
-        percent: options.percent,
-        message: JSON.stringify({
-            latest: {
-                categoryMessage: options.categoryMessage,
-                progressMessage: options.progressMessage,
-                percentage: options.percent
-            },
-            allProgress: []
-        })
+        percentage: options.percent,
+        categoryMessage: options.categoryMessage,
+        progressMessage: options.progressMessage,
+        systemFailure: options.systemFailure,
+        errorMessage: options.errorMessage
     };
@@ -372,4 +458,87 @@ }
 
+type SparkKnowledge = {
+    scopes: ScopeSelections;
+    selected: SparkKnowledgeSelection[];
+};
+
+type ScopeSelections = {
+    shared?: 'netlify-known' | 'public';
+    orgId?: string;
+    accountId?: string;
+    siteId?: string;
+};
+
+type SparkKnowledgeSelection = {
+    type: string;
+    scopeType?: keyof ScopeSelections;
+    id?: string;
+};
+
+function getKnowledgeForInputData(inputData?: Record<string, any>): SparkKnowledge | undefined {
+    if (!inputData) {
+        return undefined;
+    }
+    const selected: SparkKnowledgeSelection[] = [];
+    if (inputData._spark_ai_voiceTone) {
+        selected.push({
+            type: 'voice-and-tone',
+            id: inputData._spark_ai_voiceTone
+        });
+    }
+    if (inputData._spark_ai_targetAudience) {
+        selected.push({
+            type: 'target-audience',
+            id: inputData._spark_ai_targetAudience
+        });
+    }
+    if (selected.length === 0) {
+        return undefined;
+    }
+    return {
+        scopes: {
+            shared: 'netlify-known'
+        },
+        selected
+    };
+}
+
+async function fetchGoogleDoc(options: CustomActionRunCommonOptions, progressCallback: ProgressCallback): Promise<string | undefined> {
+    const googleDocUrl = options.inputData?._spark_ai_googleDocUrl ?? options.inputData?._spark_ai_content;
+    const match = googleDocUrl?.match(/docs\.google\.com\/document\/d\/([^/]+)\//);
+    if (!match) {
+        return;
+    }
+    const googleDocId = match[1];
+    if (!googleDocId) {
+        return;
+    }
+
+    let googleConnection;
+    if (options.currentUser && 'connections' in options.currentUser && Array.isArray(options.currentUser.connections)) {
+        googleConnection = options.currentUser.connections.find((connection) => connection.type === 'google');
+    }
+    if (!googleConnection || !('accessToken' in googleConnection)) {
+        throw new Error('Please connect your google account');
+    }
+
+    progressCallback({
+        percent: 2,
+        categoryMessage: 'Initializing',
+        progressMessage: `Fetching from ${googleDocUrl}`
+    });
+
+    // const url = `https://docs.google.com/feeds/download/documents/export/Export?id=${documentId}&exportFormat=markdown`;
+    const url = `https://www.googleapis.com/drive/v3/files/${googleDocId}/export?mimeType=text/markdown`;
+    const googleDocResult = await axios({
+        url: url,
+        method: 'get',
+        headers: {
+            Authorization: `Bearer ${googleConnection.accessToken}`
+        }
+    });
+    return googleDocResult.data;
+}
+
 export const Actions = {
     GenerateContentFromPreset
 };

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

543969

increased by2.45%

Lines of code

10185

increased by2.77%

No dependency changes