Merge pull request #3 from Oceania2018/master

merge newest code

Author: geffzhang

.gitignore

@@ -286,21 +286,9 @@ __pycache__/
 *.btm.cs
 *.odx.cs
 *.xsd.cs
-/App_Data
-/Bot.WebStarter/App_Data/bot-rasa.db
-/Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
-/BotSharp.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
-/BotSharp.UnitTest/App_Data/DbInitializer/Agents
-/BotSharp.UnitTest/App_Data/BotSharp.db
 /BotSharp.WebHost/App_Data/BotSharp.db
 /BotSharp.UI
-/BotSharp.WebHost/App_Data/ModelFiles
-/BotSharp.WebHost/App_Data/TrainingFiles
-/BotSharp.WebHost/App_Data/PredictFiles
 /BotSharp.WebHost/App_Data/Projects
 /BotSharp.WebHost/PublishOutput
 /Data
-/docs/build
 /docs/_build
-/BotSharp.WebHost/App_Data/AgentArchive/Smart Niraj.zip
-/BotSharp.NLP.UnitTest/wordvec_enu.bin

BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs

@@ -88,13 +88,16 @@ public double CalPosteriorProb(string Y, double[] features, double priorProb, Di
         {
             int featureCount = features.Length;
 
-            double postProb = priorProb;
+            double postProb = Math.Log(priorProb);
 
             // loop features
             for (int x = 0; x < featureCount; x++)
             {
                 string key = $"{Y} f{x} {features[x]}";
-                postProb += condProbDictionary[key];
+                if(features[x] == 1)
+                {
+                    postProb += condProbDictionary[key];
+                }
             }
 
             return Math.Pow(2, postProb);

BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj

@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
     <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 

BotSharp.Core.UnitTest/Performance/Spotify.cs

@@ -0,0 +1,83 @@
+using BotSharp.Core.Agents;
+using BotSharp.Core.Engines;
+using BotSharp.Core.Engines.BotSharp;
+using BotSharp.Core.Models;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using BotSharp.Algorithm.Extensions;
+
+namespace BotSharp.Core.UnitTest.Performance
+{
+    [TestClass]
+    public class Spotify : TestEssential
+    {
+        private List<Tuple<AIRequest, string>> Samples;
+        private IBotPlatform _platform;
+
+        [TestMethod]
+        public void IntentAccuracy()
+        {
+            int correct = 0;
+            List<Tuple<string, string>> errors = new List<Tuple<string, string>>();
+
+            var agent = LoadAgent();
+
+            for (int i = 0; i < Samples.Count; i++)
+            {
+                var aIResponse = _platform.TextRequest(Samples[i].Item1);
+                if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
+                {
+                    correct++;
+                }
+                else
+                {
+                    errors.Add(new Tuple<string, string>(Samples[i].Item2, Samples[i].Item1.Query[0]));
+                }
+            }
+
+           double accuracy = correct / (Samples.Count + 0.0);
+        }
+
+        private Agent LoadAgent()
+        {
+            _platform = new BotSharpAi();
+
+            // Load agent
+            var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", "Spotify");
+            string model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last();
+            var modelPath = Path.Combine(projectPath, model);
+            var agent = _platform.LoadAgentFromFile(modelPath);
+
+            // Init samples
+            Samples = new List<Tuple<AIRequest, string>>();
+            /*agent.Corpus.UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>
+            {
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "play the 50 Great Beatles Songs playlist in Prime Music"},
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "reproduce a the track Monster by Rihanna ft Eminem"},
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music_player_control.add_favorite", Text = "add this song to my favourites"}
+            };*/
+            agent.Corpus.UserSays.ForEach(intent =>
+            {
+                Samples.Add(new Tuple<AIRequest, string>(new AIRequest
+                {
+                    AgentDir = projectPath,
+                    Model = model,
+                    Query = new String[]
+                    {
+                        intent.Text
+                    }
+                }, intent.Intent));
+            });
+
+            //Samples.Shuffle();
+
+            var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
+
+            return agent;
+        }
+    }
+}

BotSharp.Core.UnitTest/TestEssential.cs

@@ -18,9 +18,9 @@ public abstract class TestEssential
         public TestEssential()
         {
             contentRoot = $"{Directory.GetCurrentDirectory()}{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}BotSharp.WebHost{Path.DirectorySeparatorChar}";
-
+            contentRoot = Path.GetFullPath(contentRoot);
             ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
-            var settings = Directory.GetFiles(contentRoot + $"Settings{Path.DirectorySeparatorChar}", "*.json");
+            var settings = Directory.GetFiles(contentRoot + $"..{Path.DirectorySeparatorChar}Settings{Path.DirectorySeparatorChar}", "*.json");
             settings.ToList().ForEach(setting =>
             {
                 configurationBuilder.AddJsonFile(setting, optional: false, reloadOnChange: true);

BotSharp.Core/BotSharp.Core.csproj

@@ -66,8 +66,11 @@ If you feel that this project is helpful to you, please Star on the project, we
 
   <ItemGroup>
     <Compile Remove="Accounts\**" />
+    <Compile Remove="Engines\CoreNlp\**" />
     <EmbeddedResource Remove="Accounts\**" />
+    <EmbeddedResource Remove="Engines\CoreNlp\**" />
     <None Remove="Accounts\**" />
+    <None Remove="Engines\CoreNlp\**" />
   </ItemGroup>
 
   <ItemGroup>
@@ -80,10 +83,6 @@ If you feel that this project is helpful to you, please Star on the project, we
     <PackageReference Include="RestSharp" Version="106.3.1" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Engines\CoreNlp\" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\BotSharp.NLP\BotSharp.NLP.csproj" />
   </ItemGroup>

BotSharp.Core/Engines/BotEngineBase.cs

@@ -46,7 +46,7 @@ public AIResponse TextRequest(AIRequest request)
             {
                 doc.Sentences[0].Entities = new List<NlpEntity>();
             }
-            doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value));
+            doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value);
 
             return new AIResponse
             {

BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs

@@ -58,9 +58,6 @@ public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
             var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false);
 
             Console.WriteLine($"Saved model to {modelFileName}");
-            meta.Meta = new JObject();
-            meta.Meta["compiled at"] = "Aug 3, 2018";
-
 
             return true;
         }