@bespoken-api/shared - npm Package Compare versions

Comparing version 0.5.9 to 0.5.11

CHANGELOG.md

 # @bespoken-api/shared
 
+## 0.5.11
+
+### Patch Changes
+
+- 2c7432c: Bug fixes for wizard
+
+## 0.5.10
+
+### Patch Changes
+
+- 2dce213: Updates for latest version of Wizard
+
 ## 0.5.9
@@ -4,0 +16,0 @@

jsconfig.json

 {
   "extends": "../../jsconfig.json",
-  "include": ["./bin", "./lib", "./test"]
+  "include": ["./bin", "./lib", "./test", "../batch/lib/class-util.js", "../batch/lib/class-util.js"]
 }

lib/audio-util.js

@@ -52,2 +52,19 @@ const fs = require('fs')
   /**
+   * @param {Buffer} leftBuffer
+   * @param {Buffer} rightBuffer
+   * @param {number} [sampleRate=8000]
+   * @returns {Buffer}
+   */
+  static pcmToTwoChannelWav (leftBuffer, rightBuffer, sampleRate = 8000) {
+    const wav = new WaveFile()
+
+    // Convert our input buffer to an Int16Array - very important!
+    const leftInputArray = new Int16Array(leftBuffer.buffer)
+    const rightInputArray = new Int16Array(rightBuffer.buffer)
+
+    wav.fromScratch(2, sampleRate, '16', [leftInputArray, rightInputArray])
+    return Buffer.from(wav.toBuffer())
+  }
+
+  /**
    *
@@ -54,0 +71,0 @@ * @param {Number} silenceTime

lib/env.js

@@ -0,1 +1,3 @@
+const execSync = require('child_process').execSync
+const fs = require('fs')
 const logger = require('./logger')('ENV')
@@ -8,17 +10,30 @@
   /**
-   *
-   * @param {string} key
-   * @param {boolean} [defaultValue]
-   * @returns {boolean}
+   * @returns {void}
    */
-  static variableAsBoolean (key, defaultValue) {
-    const value = Env.variable(key)
-    if (value === 'TRUE' || value === 'true') {
-      return true
+  static config () {
+    if (fs.existsSync('.env')) {
+      logger.info('Loading env from local .env file')
+    } else if (fs.existsSync('encrypted.env')) {
+      logger.info('Loading env from local encrypted.env file')
+      if (!process.env.AGE_PRIVATE_KEY) {
+        console.error('Environment variables AGE_PRIVATE_KEY not set')
+        process.exit(1)
+      }
+
+      const sopsDir = `${require('os').homedir()}/.sops`
+      if (!fs.existsSync(sopsDir)) {
+        fs.mkdirSync(sopsDir)
+      }
+
+      const sopsFilePath = `${sopsDir}/bespoken.txt`
+      fs.writeFileSync(sopsFilePath, process.env.AGE_PRIVATE_KEY)
+
+      process.env.SOPS_AGE_KEY_FILE = sopsFilePath
+
+      // Uses sops to decrypt a file
+      const output = execSync('sops -d encrypted.env > .env')
+      console.info('Setting up environment: ' + output)
     }
 
-    if (defaultValue) {
-      return defaultValue
-    }
-    return false
+    require('dotenv').config()
   }
@@ -43,3 +58,3 @@
   static requiredVariable (key, defaultValue) {
-    const value = process.env[key]
+    const value = Env.variable(key)
     if (value) {
@@ -66,2 +81,10 @@ return value
   static variable (key) {
+    let value
+    // If we are running locally, look for variables ending in local first
+    if (process.env.LOCAL) {
+      value = process.env[key + '_LOCAL']
+      if (value) {
+        return value
+      }
+    }
     return process.env[key]
@@ -71,3 +94,35 @@ }
   /**
+   *
    * @param {string} key
+   * @param {boolean} [defaultValue]
+   * @returns {boolean}
+   */
+  static variableAsBoolean (key, defaultValue) {
+    const value = Env.variable(key)
+    if (value === 'TRUE' || value === 'true') {
+      return true
+    }
+
+    if (defaultValue) {
+      return defaultValue
+    }
+    return false
+  }
+
+  /**
+   *
+   * @param {string} key
+   * @returns {number | undefined}
+   */
+  static variableAsInt (key) {
+    const value = Env.variable(key)
+    if (value === undefined) {
+      return undefined
+    }
+
+    return parseInt(value)
+  }
+
+  /**
+   * @param {string} key
    * @param {string} valueToSearchFor
@@ -74,0 +129,0 @@ * @returns {boolean}

lib/ffmpeg.js

 
-const logger = require('@bespoken-api/shared/lib/logger')('FFMPEG')
+const logger = require('./logger')('FFMPEG')
 const ChildProcess = require('child_process')
@@ -22,34 +22,96 @@
   /**
+   * @param {string} command
+   * @param {import('stream').Readable} inputStream
+   * @returns {FFMPEG}
+   */
+  static stream (command, inputStream) {
+    return new FFMPEG(command, inputStream)
+  }
+
+  /**
    *
    * @param {string} command
+   * @param {import('stream').Readable} [inputStream]
    */
-  constructor (command) {
+  constructor (command, inputStream) {
     this.command = command
     this.consoleOutput = ''
+    this.inputStream = inputStream
+    /** @private @type {ChildProcess.ChildProcessWithoutNullStreams | undefined} */
+    this._childProcess = undefined
+
+    /** @private @type {(function(Buffer):void) | undefined} */
+    this._dataCallback = undefined
+
+    /** @private */
+    this._debug = false
   }
 
   /**
+   * @returns {FFMPEG}
+   */
+  enableDebug () {
+    this._debug = true
+    return this
+  }
+
+  /**
+   * @returns {void}
+   */
+  exit () {
+    if (this._childProcess && this._childProcess.exitCode !== null) {
+      this._childProcess.kill()
+    }
+  }
+
+  /**
    *
+   * @param {function(Buffer):void} callback
+   * @returns {void}
+   */
+  onData (callback) {
+    this._dataCallback = callback
+  }
+
+  /**
+   *
    * @returns {Promise<Buffer | undefined>}
    */
   async process () {
-    console.info('FFMPEG running command: ' + this.command)
+    logger.info('FFMPEG running command: ' + this.command)
     const commandArray = this.command.split(' ')
     return new Promise((resolve, reject) => {
-      const process = ChildProcess.spawn(commandArray[0], commandArray.slice(1))
+      this._childProcess = ChildProcess.spawn(commandArray[0], commandArray.slice(1))
 
-      let imageData = Buffer.alloc(0)
-      process.stdout.on('data', (data) => {
-        imageData = Buffer.concat([imageData, data])
+      let outputData = Buffer.alloc(0)
+      this._childProcess.stdout.on('data', (data) => {
+        if (this._dataCallback) {
+          this._dataCallback(data)
+        }
+        outputData = Buffer.concat([outputData, data])
       })
 
-      process.on('close', () => {
-        resolve(imageData)
+      this._childProcess.on('close', () => {
+        resolve(outputData)
       })
 
-      process.stderr.on('data', (data) => {
+      this._childProcess.stderr.on('data', (data) => {
         this.consoleOutput += data.toString()
+        if (this._debug) {
+          console.info(data.toString())
+        }
       })
 
-      process.stdin.end()
+      if (this.inputStream) {
+        this.inputStream.on('data', (data) => {
+          this._childProcess?.stdin.write(data)
+        })
+
+        this.inputStream.on('close', () => {
+          // Close stdin when our underlying stream closes, which causes the process to stop
+          this._childProcess?.stdin.end()
+        })
+      } else {
+        this._childProcess.stdin.end()
+      }
     })
@@ -86,2 +148,3 @@ }
 }
+
 module.exports = FFMPEG

lib/firebase-admin.js

@@ -6,16 +6,16 @@ const admin = require('firebase-admin')
 try {
-    logger.info('Initializing Firebase admin unique instrance at inside @bespoken-api/shared')
-    admin.initializeApp({
-        credential: admin.credential.cert({
-            privateKey: Config.env('FIREBASE_KEY').replace(/\\n/g, '\n'),
-            clientEmail: Config.env('FIREBASE_EMAIL').replace(/\\n/g, '\n'),
-            projectId: Config.env('FIREBASE_PROJECT')
-        }),
-        databaseURL: Config.env('FIREBASE_URL')
-    })
+  logger.info('Initializing Firebase admin unique instrance at inside @bespoken-api/shared')
+  admin.initializeApp({
+    credential: admin.credential.cert({
+      privateKey: Config.env('FIREBASE_KEY').replace(/\\n/g, '\n'),
+      clientEmail: Config.env('FIREBASE_EMAIL').replace(/\\n/g, '\n'),
+      projectId: Config.env('FIREBASE_PROJECT')
+    }),
+    databaseURL: Config.env('FIREBASE_URL')
+  })
 } catch (err) {
-    console.error('Error creating reference to FirebaseAdmin', err)
-    throw err
+  console.error('Error creating reference to FirebaseAdmin', err)
+  throw err
 }
 
-module.exports = { admin }
+module.exports = { admin }

lib/firestore-util.js

@@ -51,2 +51,3 @@ const Env = require('./env')
     // you can pass options as a third option - optional
+    // ts-ignore
     const data = await FirestoreExportImport.backupFromDoc(collection, documentID)
@@ -53,0 +54,0 @@ // console.log(JSON.stringify(data))

lib/json-util.js

@@ -0,1 +1,4 @@
+const _ = require('lodash')
+const logger = require('./logger')('JSONUTL')
+
 /**
@@ -7,2 +10,17 @@ *
    *
+   * @param {...string} pathParts
+   * @returns {any | undefined}
+   */
+  static fromFile (...pathParts) {
+    const fileName = require('path').join.apply(JSONUtil, pathParts)
+    try {
+      return JSON.parse(require('fs').readFileSync(fileName, 'utf-8'))
+    } catch (e) {
+      logger.error('could not parse JSON: ' + fileName)
+      return undefined
+    }
+  }
+
+  /**
+   *
    * @param {any} target
@@ -24,2 +42,32 @@ * @param {any} json
   /**
+   * @param {any} json
+   * @param {number} [indent]
+   * @returns {string}
+   */
+  static safeStringify (json, indent) {
+    return JSON.stringify(json, JSONUtil._referenceReplacer(), indent)
+  }
+
+  /**
+   * @param {any} json
+   * @param {any} [replacer]
+   * @param {number} [indent]
+   * @param {string[]} excludes
+   * @returns {string}
+   */
+  static stringify (json, replacer, indent, ...excludes) {
+    if (excludes) {
+      json = _.cloneDeep(json)
+      for (const exclude of excludes) {
+        _.set(json, exclude, undefined)
+      }
+    }
+
+    if (!replacer) {
+      replacer = JSONUtil._referenceRemover()
+    }
+    return JSON.stringify(json, replacer, indent)
+  }
+
+  /**
    *
@@ -41,8 +89,26 @@ * @param {any} o
   /**
-   * @param {any} json
-   * @param {number} [indent]
-   * @returns {string}
+   * Got this from here:
+   * https://stackoverflow.com/a/61749783
+   * @returns {function(any, string)}
    */
-  static safeStringify (json, indent) {
-    return JSON.stringify(json, JSONUtil._referenceReplacer(), indent)
+  static _referenceReplacer () {
+    const m = new Map(); const v = new Map(); let init = null
+
+    return function (field, value) {
+      const p = m.get(this) + (Array.isArray(this) ? `[${field}]` : '.' + field)
+      const isComplex = value === Object(value)
+
+      if (isComplex) m.set(value, p)
+
+      const pp = v.get(value) || ''
+      const path = p.replace(/undefined\.\.?/, '')
+
+      let val = pp ? `#REF:${pp[0] === '[' ? '$' : '$.'}${pp}` : value
+
+      // eslint-disable-next-line no-unused-expressions
+      !init ? (init = value) : (val === init ? val = '#REF:$' : 0)
+      if (!pp && isComplex) v.set(value, path)
+
+      return val
+    }
   }
@@ -53,6 +119,9 @@
    * https://stackoverflow.com/a/61749783
+   * @private
    * @returns {function(any, string)}
    */
-  static _referenceReplacer () {
-    const m = new Map(); const v = new Map(); let init = null
+  static _referenceRemover () {
+    const m = new Map()
+    const v = new Map()
+    let init = null
 
@@ -67,3 +136,3 @@ return function (field, value) {
       const path = p.replace(/undefined\.\.?/, '')
-      let val = pp ? `#REF:${pp[0] == '[' ? '$' : '$.'}${pp}` : value
+      let val = pp ? undefined : value
 
@@ -70,0 +139,0 @@ // eslint-disable-next-line no-unused-expressions

lib/json-wrapper.js

@@ -6,7 +6,32 @@ /**
   constructor (json) {
-    this.json = json
+    this.json = json || {}
   }
 
   /**
+   *
    * @param {string} key
+   * @returns {boolean}
+   */
+  boolean (key) {
+    return this.get(key) ? this.get(key) === true : false
+  }
+
+  /**
+   * @param {string} key
+   * @returns {any | undefined}
+   */
+  get (key) {
+    return this.json[key]
+  }
+
+  /**
+   * @param {string} key
+   * @returns {boolean}
+   */
+  has (key) {
+    return this.json[key] !== undefined
+  }
+
+  /**
+   * @param {string} key
    * @returns {number | undefined}
@@ -13,0 +38,0 @@ */

lib/logger.js

@@ -68,2 +68,7 @@ const _ = require('lodash')
     if (this.isDebugEnabled()) {
+      if (!require('fs').existsSync(fileName)) {
+        const dir = fileName.split('/')
+        dir.pop()
+        require('fs').mkdirSync(dir.join('/'), { recursive: true })
+      }
       return require('fs').writeFileSync(fileName, buffer)
@@ -193,4 +198,4 @@ }
 if (process.env.SILENT) {
-  console.timeEnd = () => {}
+  console.timeEnd = () => { }
 }
 module.exports = Logger.logger

lib/s3-util.js

 const logger = require('./logger')('S3UTIL')
+const Env = require('./env')
 const GetObjectCommand = require('@aws-sdk/client-s3').GetObjectCommand
@@ -6,2 +7,3 @@ const ListObjectsV2Command = require('@aws-sdk/client-s3').ListObjectsV2Command
 const S3Client = require('@aws-sdk/client-s3').S3Client
+const Upload = require('@aws-sdk/lib-storage').Upload
 
@@ -106,2 +108,3 @@ /**
   static async put (bucket, key, buffer, mimeType) {
+    Env.requiredVariable('AWS_ACCESS_KEY_ID')
     const client = new S3Client({ region: 'us-east-1' })
@@ -116,3 +119,3 @@ try {
 
-      logger.info('putting object: ' + key)
+      logger.debug('putting object: ' + key)
       await client.send(putObjectCommand)
@@ -140,2 +143,36 @@ // process data.
   /**
+   * @param {import('stream').Readable} readable
+   * @param {string} mimeType
+   * @param {string} bucket
+   * @param {string} key
+
+   * @returns {Promise<void>}
+   */
+  static async upload (readable, mimeType, bucket, key) {
+    Env.requiredVariable('AWS_ACCESS_KEY_ID')
+    const client = new S3Client({ region: 'us-east-1' })
+
+    const params = {
+      Body: readable,
+      Bucket: bucket,
+      // ContentType: mimeType,
+      Key: key
+    }
+
+    const parallelUploads3 = new Upload({
+      client: client,
+      leavePartsOnError: false, // optional manually handle dropped parts
+      params: params,
+      partSize: 1024, // optional size of each part, in bytes, at least 5MB
+      queueSize: 4 // optional concurrency configuration
+    })
+
+    parallelUploads3.on('httpUploadProgress', (progress) => {
+      console.log(progress)
+    })
+
+    await parallelUploads3.done()
+  }
+
+  /**
    * @private
@@ -142,0 +179,0 @@ * @param {import('stream').Readable} stream

lib/util.js

 const Config = require('./config')
 const crypto = require('crypto')
 const FS = require('fs')
+const logger = require('./logger')('MUTEX')
 const Path = require('path')
@@ -92,6 +93,15 @@ const Readable = require('stream').Readable
   static async mutex (name) {
-    return Mutex.acquire(name)
+    return /** @type {Promise<Mutex>} */ (Mutex.acquire(name))
   }
 
   /**
+   * @param {string} name
+   * @param {number} maximumWaitTime
+   * @returns {Promise<Mutex | undefined>}
+   */
+  static async mutexOrTimeout (name, maximumWaitTime) {
+    return Mutex.acquire(name, maximumWaitTime)
+  }
+
+  /**
    * @param {number} time in ms
@@ -101,2 +111,5 @@ * @returns {Promise<void>}
   static async sleep (time) {
+    if (time === 0) {
+      return Promise.resolve()
+    }
     return new Promise((resolve) => {
@@ -140,12 +153,28 @@ setTimeout(() => {
    * @param {string} name
-   * @returns {Promise<Mutex>}
+   * @param {number} [maximumWaitTime]
+   * @returns {Promise<Mutex | undefined>}
    */
-  static async acquire (name) {
-    let mutex = Mutex.Locks[name]
+  static async acquire (name, maximumWaitTime) {
+    let mutex = Mutex.Map[name]
     if (mutex) {
-      await mutex.waitFor()
-      mutex.lock()
+      logger.debug('waiting for mutex: ' + name)
+      const startTime = new Date().getTime()
+
+      while (true) {
+        const availableMutex = await mutex.waitFor()
+        if (availableMutex && !availableMutex.locked) {
+          logger.debug('acquired mutex: ' + name)
+
+          availableMutex.lock()
+          return availableMutex
+        } else {
+          if (maximumWaitTime && new Date().getTime() - startTime > maximumWaitTime) {
+            return undefined
+          }
+        }
+      }
     } else {
       mutex = new Mutex(name)
-      Mutex.Locks[name] = mutex
+      logger.debug('creating mutex: ' + name)
+      Mutex.Map[name] = mutex
     }
@@ -160,3 +189,5 @@
   constructor (name) {
+    this.locked = false
     this.name = name
+    this.waitingResolves = []
     this.lock()
@@ -169,6 +200,3 @@ }
   lock () {
-    /** @type {Promise<Mutex>} */
-    this.onReleasePromise = new Promise((resolve) => {
-      this.onRelease = resolve
-    })
+    this.locked = true
   }
@@ -180,5 +208,8 @@
   release () {
-    delete Mutex.Locks[this.name]
-    if (this.onRelease) {
-      this.onRelease(this)
+    logger.debug('releasing mutex: ' + this.name)
+    this.locked = false
+    const resolveList = this.waitingResolves
+    this.waitingResolves = []
+    for (const resolve of resolveList) {
+      resolve(this)
     }
@@ -188,14 +219,15 @@ }
   /**
-   * @returns {Promise<Mutex>}
+   * @returns {Promise<Mutex | undefined>}
    */
   async waitFor () {
-    if (this.onReleasePromise) {
-      await this.onReleasePromise
+    if (!this.locked) {
+      return this
     }
-
-    return this
+    return new Promise((resolve, reject) => {
+      this.waitingResolves.push(resolve)
+    })
   }
 }
 
-Mutex.Locks = {}
+Mutex.Map = {}
 module.exports = Util

lib/video.js

 const FFMPEG = require('./ffmpeg')
 const FS = require('fs')
-const logger = require('@bespoken-api/shared/lib/logger')('VIDEO')
+const logger = require('./logger')('VIDEO')
 const UUID = require('uuid')
@@ -5,0 +5,0 @@

lib/yaml-templates.js

@@ -0,1 +1,4 @@
+const { set } = require("lodash")
+const { v4 } = require("uuid")
+
 /**
@@ -6,7 +9,12 @@ * @param platform
  */
-function getYamlConfigurationTemplate (platform) {
+function getYamlConfigurationTemplate(platform) {
   const defaultConfig = {
-    locale: 'en-US',
     platform,
-    type: 'e2e'
+    virtualDeviceToken: null,
+    type: "e2e",
+    locale: "en-US",
+    voiceId: "Joey",
+    projectId: v4(),
+    trace: true,
+    asyncMode: true
   }
@@ -16,11 +24,23 @@
     defaultConfig["virtualDeviceConfig"] = {
-      "nlu": {
-        "serviceUrl": "",
-        "apiKey": "",
-        "assistantId": ""
-      }
+      "serviceUrl": "",
+      "apiKey": "",
+      "assistantId": ""
     }
+  } else if (platform === 'webchat') {
+    defaultConfig["extraParameters"] = {
+      initialize: '$("div#btn-open-chat").click()',
+      headless: true,
+      inputSelector: '.text_chat',
+      replySelector: '.reply_selector',
+      timeoutOnReply: 15000,
+      openSelectors: ['.widget_div'],
+      timeoutOnLoad: 10000,
+      timeoutOnInitialize: 5000,
+    }
+    defaultConfig["maxAsyncE2EResponseWaitTime"] = 200000
+  } else if (platform === 'phone') {
+    defaultConfig["maxAsyncE2EResponseWaitTime"] = 60000
+    set(defaultConfig, 'virtualDeviceConfig.speechModel', "phone_call")
   }
 
-  
   return defaultConfig
@@ -35,3 +55,3 @@ }
  */
-function getYamlTemplate (name, platform) {
+function getYamlTemplate(name, platform) {
   const yaml = `---
@@ -49,3 +69,3 @@ - test : Test 1
  */
-function getInteraction (name, platform) {
+function getInteraction(name, platform) {
   let interaction = '  - hello : hi! how can I help?'
@@ -56,3 +76,2 @@ if (['twilio', 'phone'].indexOf(platform) > -1) {
     interaction += `    - prompt : welcome to ${name}\n`
-    interaction += '    - set finishOnPhrase : what can I help you with?'
   } else if (platform === 'google') {
@@ -59,0 +78,0 @@ interaction = `  - talk to ${name} : welcome to ${name}`

package.json

@@ -8,5 +8,10 @@ {
   },
+  "bin": {
+    "prisma-util": "./lib/prisma-util.js"
+  },
   "dependencies": {
     "@aws-sdk/client-ecs": "^3.52.0",
     "@aws-sdk/client-s3": "^3.42.0",
+    "@aws-sdk/client-ses": "^3.348.0",
+    "@aws-sdk/lib-storage": "^3.321.1",
     "@google-cloud/firestore": "^5.0.1",
@@ -33,3 +38,3 @@ "axios": "^0.21.4",
   },
-  "version": "0.5.9",
+  "version": "0.5.11",
   "scripts": {
@@ -36,0 +41,0 @@ "compile": "tsc --project jsconfig.json",

New alerts

Shell access

Supply chain risk

This module accesses the system shell. Accessing the system shell increases the risk of executing arbitrary code.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 5 instances in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

67719

increased by26.59%

Number of package files

34

increased by13.33%

Lines of code

2296

increased by24.18%

Worsened metrics

Dependency count

14

increased by16.67%

Number of low supply chain risk alerts

44

increased by46.67%

Number of medium supply chain risk alerts

4

increased by300%

Dependency changes

• + Added@aws-sdk/client-ses@^3.348.0

• + Added@aws-sdk/lib-storage@^3.321.1

• + Added@aws-sdk/client-ses@3.606.0(transitive)

• + Added@aws-sdk/lib-storage@3.608.0(transitive)

• + Addedbuffer@5.6.0(transitive)

• + Addedevents@3.3.0(transitive)

• + Addedieee754@1.2.1(transitive)

• + Addedstream-browserify@3.0.0(transitive)