Feature/version3 4 (axa-group#227)

* feat: remove handlebars

* feat: Adding default intent and score when score is less than threshold

* feat: update deps

* feat: using decay learning rate

* test: adjust tests

Author: Jesús Seijas

lib/classifiers/neural-network.js

@@ -150,14 +150,17 @@ class NeuralNetwork {
     const { learningRate, leakyReluAlpha: alpha, momentum } = this.trainOpts;
     const numOutputs = this.sizes[1];
     let error = 0;
+    const decayLearningRate =
+      learningRate / (1 + 0.001 * this.status.iterations);
     for (let node = 0, l = numOutputs; node < l; node += 1) {
       const perceptron = this.perceptrons[node];
       const { changes, weights } = perceptron;
       const output = outputs[node];
       const currentError = target[node] - output;
       if (currentError) {
         error += currentError ** 2;
-        const delta = (output > 0 ? 1 : alpha) * currentError * learningRate;
+        const delta =
+          (output > 0 ? 1 : alpha) * currentError * decayLearningRate;
         for (let k = 0; k < incoming.length; k += 1) {
           const change = delta * incoming[k] + momentum * changes[k];
           changes[k] = change;

lib/index.js

@@ -51,7 +51,7 @@ const {
 } = require('./ner');
 const { SentimentAnalyzer, SentimentManager } = require('./sentiment');
 const { SlotManager } = require('./slot');
-const { Evaluator, SimilarSearch } = require('./util');
+const { Evaluator, Handlebars, SimilarSearch } = require('./util');
 const { XTableUtils, XTable, XDoc } = require('./xtables');
 const {
   ConversationContext,
@@ -84,6 +84,7 @@ const exportClasses = {
   SentimentManager,
   SlotManager,
   Evaluator,
+  Handlebars,
   SimilarSearch,
   XTableUtils,
   XTable,

lib/nlp/nlp-manager.js

@@ -22,7 +22,7 @@
  */
 
 const fs = require('fs');
-const Handlebars = require('handlebars');
+const { Handlebars } = require('../util');
 const NluManager = require('../nlu/nlu-manager');
 const NerManager = require('../ner/ner-manager');
 const { SentimentManager } = require('../sentiment');
@@ -61,6 +61,12 @@ class NlpManager {
     this.sentiment = new SentimentManager(this.settings.sentiment);
     this.slotManager = new SlotManager();
     this.utteranceDict = this.settings.utteranceDict || { '?': 'help' };
+    this.defaultThreshold = this.settings.defaultThreshold;
+    if (this.settings.defaultThreshold === undefined) {
+      this.defaultThreshold = 0.5;
+    }
+    this.defaultIntent = this.settings.defaultIntent || 'None';
+    this.defaultScore = this.settings.defaultScore || 1;
     if (this.settings.languages) {
       this.addLanguage(this.settings.languages);
     }
@@ -392,6 +398,13 @@ class NlpManager {
         result = optional;
       }
     }
+    if (
+      this.settings.defaultThreshold > 0 &&
+      result.score < this.settings.defaultThreshold
+    ) {
+      result.intent = this.defaultIntent;
+      result.score = this.defaultScore;
+    }
     result.entities = await this.nerManager.findEntities(
       result.utterance,
       result.localeIso2,

lib/nlp/nlp-util.js

@@ -351,6 +351,39 @@ NlpUtil.useAlternative = {
   tr: false,
 };
 
+NlpUtil.useNoneFeature = {
+  en: true,
+  fa: false,
+  fr: true,
+  ru: true,
+  es: true,
+  gl: true,
+  it: true,
+  nl: true,
+  no: true,
+  pt: true,
+  pl: true,
+  sv: true,
+  tl: true,
+  id: true,
+  ja: false,
+  ar: false,
+  hy: false,
+  eu: true,
+  ca: true,
+  cs: true,
+  da: true,
+  fi: true,
+  de: true,
+  hu: true,
+  ga: true,
+  ro: true,
+  sl: true,
+  ta: false,
+  tr: true,
+  zh: false,
+};
+
 NlpUtil.tokenizers = {};
 
 module.exports = NlpUtil;

lib/nlu/brain-nlu.js

@@ -23,6 +23,7 @@
 
 const BaseNLU = require('./base-nlu');
 const BrainClassifier = require('../classifiers/brain-classifier');
+const NlpUtil = require('../nlp/nlp-util');
 
 /**
  * Class for the Logistic Regression NLU
@@ -34,6 +35,10 @@ class BrainNLU extends BaseNLU {
    */
   constructor(settings) {
     super(settings);
+    if (this.settings.useNoneFeature === undefined) {
+      this.settings.useNoneFeature =
+        NlpUtil.useNoneFeature[this.settings.language] || false;
+    }
     this.classifier = this.settings.classifier || new BrainClassifier(settings);
   }
 

lib/nlu/domain-manager.js

@@ -43,7 +43,7 @@ class DomainManager {
         : this.settings.useMasterDomain;
     this.trainByDomain =
       this.settings.trainByDomain === undefined
-        ? true
+        ? false
         : this.settings.trainByDomain;
     this.stemmer = this.settings.stemmer || NlpUtil.getStemmer(this.language);
     this.keepStopwords =
@@ -242,53 +242,6 @@ class DomainManager {
     return this.getClassifications(stems, 'master_domain');
   }
 
-  // getClassifications(utterance) {
-  //   const stems = this.tokenizeAndStem(utterance);
-  //   if (this.useStemDict) {
-  //     const stemKey = this.generateStemKey(stems);
-  //     if (this.stemDict[stemKey]) {
-  //       const classifications = [];
-  //       classifications.push({
-  //         label: this.stemDict[stemKey].intent,
-  //         value: 1,
-  //       });
-  //       Object.keys(this.intentDict).forEach(intent => {
-  //         if (intent !== this.stemDict[stemKey].intent) {
-  //           if (
-  //             !this.trainByDomain ||
-  //             this.stemDict[stemKey].domain === this.intentDict[intent]
-  //           ) {
-  //             classifications.push({ label: intent, value: 0 });
-  //           }
-  //         }
-  //       });
-  //       return { domain: this.stemDict[stemKey].domain, classifications };
-  //     }
-  //   }
-  //   if (this.trainByDomain) {
-  //     let domain;
-  //     if (Object.keys(this.domains).length > 2) {
-  //       const master = this.addDomain('master_domain');
-  //       const domainClassifications = master.getClassifications(stems);
-  //       domain = domainClassifications[0].label;
-  //     } else {
-  //       [domain] = Object.keys(this.domains).filter(x => x !== 'master_domain');
-  //     }
-  //     if (domain) {
-  //       const nlu = this.addDomain(domain);
-  //       const classifications = nlu.getClassifications(stems);
-  //       return { domain, classifications };
-  //     }
-  //     return { domain, classifications: [] };
-  //   }
-  //   const nlu = this.addDomain('master_domain');
-  //   const classifications = nlu.getClassifications(stems);
-  //   return {
-  //     domain: this.intentDict[classifications[0].label],
-  //     classifications,
-  //   };
-  // }
-
   /**
    * Exports object properties.
    * @returns {Object} Object properties.

lib/util/handlebars.js

@@ -0,0 +1,20 @@
+const Evaluator = require('./evaluator');
+
+const evaluator = new Evaluator();
+
+const dictionary = {};
+
+function compile(str) {
+  if (dictionary[str] === undefined) {
+    dictionary[str] = str.match(/{{\s*([^}]+)\s*}}/g) || [];
+  }
+  const matches = dictionary[str];
+  return (context = {}) => {
+    return matches.reduce((p, c) => {
+      const solution = evaluator.evaluate(c.substr(2, c.length - 4), context);
+      return solution ? p.replace(c, solution) : p;
+    }, str);
+  };
+}
+
+module.exports = { compile };

lib/util/index.js

@@ -23,8 +23,10 @@
 
 const SimilarSearch = require('./similar-search');
 const Evaluator = require('./evaluator');
+const Handlebars = require('./handlebars');
 
 module.exports = {
   SimilarSearch,
   Evaluator,
+  Handlebars,
 };