@telefonica/language-model-converter - npm Package Compare versions

Comparing version 2.2.1 to 2.3.0

lib/parser.js

 "use strict";
 const fs = require('fs');
 const yaml = require('js-yaml');
+const _ = require('lodash');
 class LanguageModelParser {
@@ -34,3 +35,5 @@ constructor() {
         let keys = Object.keys(this.doc);
-        let intentNames = keys.filter(intentName => !intentName.startsWith('list.'))
+        let intentNames = keys
+            .filter(intentName => !intentName.startsWith('list.'))
+            .filter(intentName => !intentName.startsWith('phraselist'))
             .map(intentName => {
@@ -42,6 +45,6 @@ if (intentName.length > 50) {
         });
-        let synonyms = new Map();
+        let replacements = new Map();
         keys.filter(intentName => intentName.startsWith('list.'))
             .forEach(intentName => {
-            synonyms.set(intentName.slice('list.${'.length, -1), this.doc[intentName]);
+            replacements.set(intentName.slice('list.${'.length, -1), this.doc[intentName]);
         });
@@ -53,3 +56,3 @@ let entitiesMap = new Map();
             sentences
-                .map((sentence) => this.expandVariables(sentence, synonyms))
+                .map((sentence) => this.expandVariables(sentence, replacements))
                 .reduce((a, b) => a.concat(b))
@@ -62,15 +65,17 @@ .forEach((sentence) => {
         });
-        luisModel.utterances = Array.from(utterances.values());
-        luisModel.entities = Array.from(entitiesMap.values());
-        luisModel.intents = intentNames.map(intent => ({ name: intent }));
-        luisModel.model_features = Array.from(synonyms.entries()).map(entry => {
-            let name = entry[0];
-            let words = entry[1];
+        let features = _.toPairs(this.doc.phraselist)
+            .map(value => {
+            let name = String(value[0]);
+            let words = value[1].words || {};
             return {
-                name,
-                words: words.map(word => this.tokenize(word).join(' ')).join(','),
+                activated: true,
                 mode: true,
-                activated: true
+                name: name,
+                words: words.map(word => this.tokenize(word).join(' ')).join(',')
             };
         });
+        luisModel.utterances = Array.from(utterances.values());
+        luisModel.entities = Array.from(entitiesMap.values());
+        luisModel.intents = intentNames.map(intent => ({ name: intent }));
+        luisModel.model_features = features;
         return luisModel;
@@ -82,13 +87,15 @@ }
             variables.forEach((values, key) => {
-                let search = '${' + key + '}';
-                if (sentence.indexOf(search) !== -1) {
-                    let newSentence = sentence.replace(search, values[0]);
-                    if (newSentence !== sentence) {
-                        expandedSentences.add(newSentence);
-                        expandedSentences.delete(sentence);
+                values.forEach(value => {
+                    let search = '${' + key + '}';
+                    if (sentence.indexOf(search) !== -1) {
+                        let newSentence = sentence.replace(search, value);
+                        if (newSentence !== sentence) {
+                            expandedSentences.add(newSentence);
+                            expandedSentences.delete(sentence);
+                        }
+                        else {
+                            expandedSentences.add(sentence);
+                        }
                     }
-                    else {
-                        expandedSentences.add(sentence);
-                    }
-                }
+                });
             });
@@ -95,0 +102,0 @@ });

lib/parser.spec.js

@@ -203,18 +203,3 @@ "use strict";
         ];
-        let expectedModelFeatures = [
-            {
-                name: 'strange',
-                words: `hola - caracola,Hola _ caracola,I ' d like,2º floor,Que tAl`,
-                activated: true,
-                mode: true
-            },
-            {
-                name: 'numbers',
-                words: `123,123 123`,
-                activated: true,
-                mode: true
-            }
-        ];
         chai_1.expect(luisModel.utterances).to.eql(expectedUtterances);
-        chai_1.expect(luisModel.model_features).to.eql(expectedModelFeatures);
     });
@@ -255,2 +240,13 @@ it('should deal with locale specifities', function () {
             {
+                'text': 'this is the country france',
+                'intent': 'my.test.intent',
+                'entities': [
+                    {
+                        'entity': 'country',
+                        'startPos': 4,
+                        'endPos': 4
+                    }
+                ]
+            },
+            {
                 'text': 'this is the country spain',
@@ -261,2 +257,7 @@ 'intent': 'my.test.expansion',
             {
+                'text': 'this is the country france',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
                 'text': 'this is the color red',
@@ -267,2 +268,7 @@ 'intent': 'my.test.expansion',
             {
+                'text': 'this is the color blue',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
                 'text': 'this is the red spain',
@@ -273,22 +279,56 @@ 'intent': 'my.test.expansion',
             {
+                'text': 'this is the blue spain',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
+                'text': 'this is the red france',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
+                'text': 'this is the blue france',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
                 'text': 'this is the spain spain',
                 'intent': 'my.test.expansion',
                 'entities': []
+            },
+            {
+                'text': 'this is the spain france',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
+                'text': 'this is the france spain',
+                'intent': 'my.test.expansion',
+                'entities': []
+            },
+            {
+                'text': 'this is the france france',
+                'intent': 'my.test.expansion',
+                'entities': []
             }
         ];
+        chai_1.expect(luisModel.utterances).to.eql(expectedUtterances);
+    });
+    it('should parse phraselists', function () {
+        let parser = new parser_1.LanguageModelParser();
+        let luisModel = parser.parse(['./test/fixtures/en-features.yaml'], 'en-us');
         let expectedModelFeatures = [
             {
-                name: 'country',
-                words: 'Spain,France',
-                mode: true,
-                activated: true
+                name: 'strange',
+                words: `hola - caracola,Hola _ caracola,I ' d like,2º floor,Que tAl`,
+                activated: true,
+                mode: true
             },
             {
-                name: 'colors',
-                words: 'Red,Blue',
-                mode: true,
-                activated: true
+                name: 'numbers',
+                words: `123,123 123`,
+                activated: true,
+                mode: true
             }
         ];
-        chai_1.expect(luisModel.utterances).to.eql(expectedUtterances);
         chai_1.expect(luisModel.model_features).to.eql(expectedModelFeatures);
@@ -295,0 +335,0 @@ });

package.json

 {
   "name": "@telefonica/language-model-converter",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "description": "Language model converter for yot-bot",
@@ -32,3 +32,4 @@ "license": "UNLICENSED",
     "glob": "^7.1.0",
-    "js-yaml": "^3.6.1"
+    "js-yaml": "^3.6.1",
+    "lodash": "^4.16.4"
   },
@@ -40,2 +41,3 @@ "devDependencies": {
     "@types/js-yaml": "^3.5.28",
+    "@types/lodash": "^4.14.37",
     "@types/mocha": "^2.2.32",
@@ -42,0 +44,0 @@ "@types/node": "^6.0.40",

README.md

@@ -12,3 +12,3 @@ # language-model-converter
 
-list.${example}:  # Defines a example list to make substitutions in the utterances
+list.${examples}:  # Defines a example list to make substitutions in the utterances
   - heaven
@@ -19,10 +19,30 @@ - hell
   - Tell me about the purgatory # Simple utterance
-  - What is ${example} # Substitution with list: Will generate for you "What is heaven" and "What is hell"
+  - What is ${examples} # Substitution with list: Will generate for you "What is heaven" and "What is hell"
 
 tef.intent.go: # Defines a Luis Intent
   - Go to [purgatory:tef.places] # Defines an entity "tef.places" giving "purgatory" as an example. The example is mandatory
-  - Head to [${example}:tef.places] # You can make substitution in the entity examples too!
+  - Head to [${examples}:tef.places] # You can make substitution in the entity examples too!
+  
+phraselist:
+  aksforinfo: # ex: How to ask for something synonims: 
+    words: 
+      - Tell me about
+      - What is
 
+  tef.places: # ex: Place examples here for training entities
+    words: 
+      - heaven
+      - hell
+      - purgatory
+      - home
+      
 ```
 
+### Limitations
+As of writing this doc, the following limitations apply in LUIS Service
+ * Max of 20 intents (`tef.intent.info` + `tef.intent.go` = 2)
+ * Max of 50 chars for the intent name (`tef.intent.info` = 15)
+ * Max of 10 phraselists (`askforinfo` + `tef.places` = 2)
+ * Max of 10 entities ([something:`tef.places`]) _You can provide as much examples as you want for the entities_
+
 ## Usage examples
@@ -29,0 +49,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

41529

increased by8.51%

Lines of code

613

increased by8.3%

Number of lines in readme file

71

increased by39.22%

Worsened metrics

Dependency count

4

increased by33.33%

Dev dependency count

19

increased by5.56%

Dependency changes

• + Addedlodash@^4.16.4

• + Addedlodash@4.17.21(transitive)