@telefonica/language-model-converter - npm Package Compare versions

Comparing version 2.1.1 to 2.2.0

lib/luis-model.d.ts

@@ -12,3 +12,3 @@ export declare namespace Luis {
         actions: any[];
-        model_features: any[];
+        model_features: ModelFeature[];
         regex_features: any[];
@@ -22,2 +22,8 @@ utterances: Utterance[];
     }
+    interface ModelFeature {
+        name: string;
+        mode: boolean;
+        words: string;
+        activated: boolean;
+    }
     interface EntityPosition {
@@ -24,0 +30,0 @@ entity: string;

lib/luis-model.js

@@ -6,3 +6,4 @@ "use strict";
     ;
+    ;
 })(Luis = exports.Luis || (exports.Luis = {}));
 //# sourceMappingURL=luis-model.js.map

lib/parser.d.ts

@@ -11,4 +11,5 @@ import { Luis } from './luis-model';
     private normalizeSentence(sentence);
-    private static wordCount(sentence);
+    private wordCount(sentence);
+    private tokenize(sentence);
     private buildUtterance(sentence, intent);
 }

lib/parser.js

@@ -34,4 +34,3 @@ "use strict";
         let keys = Object.keys(this.doc);
-        let intentNames = keys
-            .filter(intentName => !intentName.startsWith('list.'))
+        let intentNames = keys.filter(intentName => !intentName.startsWith('list.'))
             .map(intentName => {
@@ -43,13 +42,13 @@ if (intentName.length > 50) {
         });
-        let list = new Map();
-        keys
-            .filter(intentName => intentName.startsWith('list.'))
+        let synonyms = new Map();
+        keys.filter(intentName => intentName.startsWith('list.'))
             .forEach(intentName => {
-            list.set(intentName.slice('list.${'.length, -1), this.doc[intentName]);
+            synonyms.set(intentName.slice('list.${'.length, -1), this.doc[intentName]);
         });
         let entitiesMap = new Map();
+        let utterances = new Set();
         intentNames.forEach(intent => {
             let sentences = this.doc[intent];
             sentences
-                .map((sentence) => this.expandVariables(sentence, list))
+                .map((sentence) => this.expandVariables(sentence, synonyms))
                 .reduce((a, b) => a.concat(b))
@@ -59,7 +58,18 @@ .forEach((sentence) => {
                 utterance.entities.forEach(entity => this.registerEntity(entity, entitiesMap));
-                luisModel.utterances.push(utterance);
+                utterances.add(utterance);
             });
         });
+        luisModel.utterances = Array.from(utterances.values());
         luisModel.entities = Array.from(entitiesMap.values());
         luisModel.intents = intentNames.map(intent => ({ name: intent }));
+        luisModel.model_features = Array.from(synonyms.entries()).map(entry => {
+            let name = entry[0];
+            let words = entry[1];
+            return {
+                name,
+                words: words.map(word => this.tokenize(word).join(' ')).join(','),
+                mode: true,
+                activated: true
+            };
+        });
         return luisModel;
@@ -71,15 +81,13 @@ }
             variables.forEach((values, key) => {
-                values.forEach(value => {
-                    let search = '${' + key + '}';
-                    if (sentence.indexOf(search) !== -1) {
-                        let newSentence = sentence.replace(search, value);
-                        if (newSentence !== sentence) {
-                            expandedSentences.add(newSentence);
-                            expandedSentences.delete(sentence);
-                        }
-                        else {
-                            expandedSentences.add(sentence);
-                        }
+                let search = '${' + key + '}';
+                if (sentence.indexOf(search) !== -1) {
+                    let newSentence = sentence.replace(search, values[0]);
+                    if (newSentence !== sentence) {
+                        expandedSentences.add(newSentence);
+                        expandedSentences.delete(sentence);
                     }
-                });
+                    else {
+                        expandedSentences.add(sentence);
+                    }
+                }
             });
@@ -134,3 +142,6 @@ });
     }
-    static wordCount(sentence) {
+    wordCount(sentence) {
+        return this.tokenize(sentence).length;
+    }
+    tokenize(sentence) {
         return sentence
@@ -143,4 +154,3 @@ .replace(/[^\w\u00C0-\u017F]/g, capture => ` ${capture} `)
             .trim()
-            .split(' ')
-            .length;
+            .split(' ');
     }
@@ -157,5 +167,5 @@ buildUtterance(sentence, intent) {
                 extractedEntities.forEach(entity => {
-                    let startPos = LanguageModelParser.wordCount(parts);
+                    let startPos = this.wordCount(parts);
                     parts += entity.entityValue;
-                    let endPos = LanguageModelParser.wordCount(parts) - 1;
+                    let endPos = this.wordCount(parts) - 1;
                     entities.push({
@@ -162,0 +172,0 @@ entity: entity.entityType,

lib/parser.spec.js

@@ -203,3 +203,12 @@ "use strict";
         ];
+        let expectedModelFeatures = [
+            {
+                name: 'strange',
+                words: `hola - caracola,Hola _ caracola,I ' d like,2º floor,Que tAl`,
+                activated: true,
+                mode: true
+            }
+        ];
         chai_1.expect(luisModel.utterances).to.eql(expectedUtterances);
+        chai_1.expect(luisModel.model_features).to.eql(expectedModelFeatures);
     });
@@ -240,13 +249,2 @@ it('should deal with locale specifities', function () {
             {
-                'text': 'this is the country france',
-                'intent': 'my.test.intent',
-                'entities': [
-                    {
-                        'entity': 'country',
-                        'startPos': 4,
-                        'endPos': 4
-                    }
-                ]
-            },
-            {
                 'text': 'this is the country spain',
@@ -257,7 +255,2 @@ 'intent': 'my.test.expansion',
             {
-                'text': 'this is the country france',
-                'intent': 'my.test.expansion',
-                'entities': []
-            },
-            {
                 'text': 'this is the color red',
@@ -268,7 +261,2 @@ 'intent': 'my.test.expansion',
             {
-                'text': 'this is the color blue',
-                'intent': 'my.test.expansion',
-                'entities': []
-            },
-            {
                 'text': 'this is the red spain',
@@ -279,40 +267,25 @@ 'intent': 'my.test.expansion',
             {
-                'text': 'this is the blue spain',
-                'intent': 'my.test.expansion',
-                'entities': []
-            },
-            {
-                'text': 'this is the red france',
-                'intent': 'my.test.expansion',
-                'entities': []
-            },
-            {
-                'text': 'this is the blue france',
-                'intent': 'my.test.expansion',
-                'entities': []
-            },
-            {
                 'text': 'this is the spain spain',
                 'intent': 'my.test.expansion',
                 'entities': []
-            },
+            }
+        ];
+        let expectedModelFeatures = [
             {
-                'text': 'this is the spain france',
-                'intent': 'my.test.expansion',
-                'entities': []
+                name: 'country',
+                words: 'Spain,France',
+                mode: true,
+                activated: true
             },
             {
-                'text': 'this is the france spain',
-                'intent': 'my.test.expansion',
-                'entities': []
-            },
-            {
-                'text': 'this is the france france',
-                'intent': 'my.test.expansion',
-                'entities': []
+                name: 'colors',
+                words: 'Red,Blue',
+                mode: true,
+                activated: true
             }
         ];
         chai_1.expect(luisModel.utterances).to.eql(expectedUtterances);
+        chai_1.expect(luisModel.model_features).to.eql(expectedModelFeatures);
     });
 });
 //# sourceMappingURL=parser.spec.js.map

package.json

 {
   "name": "@telefonica/language-model-converter",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "description": "Language model converter for yot-bot",
@@ -5,0 +5,0 @@ "license": "UNLICENSED",

README.md

 # language-model-converter
 
-This is a tool to convert language files defined in [language-model](../language-model) into the LUIS internal JSON representation.
+This is a tool to convert language files defined in yaml language model into the (LUIS)(http://luis.ai) internal JSON representation.
 
+### The language model
+This is a [yaml](http://www.yaml.org/) format file that is conveted to a [JSON file](https://dev.projectoxford.ai/docs/services/56d95961e597ed0f04b76e58/operations/56f8a55119845511c81de480) that LUIS can undestand
+
+Words preceed by `#` are comments, and its purpose is to leave some insights to people that will help on understanding whats going on. These words will be removed when training luis, so feel free to add as much as you want to help next colleagues that will manage the file
+
+```yaml
+
+list.${example}:  # Defines a example list to make substitutions in the utterances
+  - heaven
+  - hell
+  
+tef.intent.info: # Defines a Luis Intent
+  - Tell me about the purgatory # Simple utterance
+  - What is ${example} # Substitution with list: Will generate for you "What is heaven" and "What is hell"
+
+tef.intent.go: # Defines a Luis Intent
+  - Go to [purgatory:tef.places] # Defines an entity "tef.places" giving "purgatory" as an example. The example is mandatory
+  - Head to [${example}:tef.places] # You can make substitution in the entity examples too!
+
+```
+
 ## Usage examples
@@ -27,2 +48,4 @@
 
-Notice that you can use [luis-cli](../luis-cli) to import or update the generated LUIS models.
+Notice that you can use [luis-cli](../luis-cli) to import or update the generated LUIS models.
+
+

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

37994

increased by2.91%

Number of lines in readme file

51

increased by88.89%

Worsened metrics

Lines of code

560

decreased by-1.58%

No dependency changes