Merge pull request #596 from iceljc/bugfix/refine-knowledge-base

unite knowledge search model

Author: iceljc

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -3,8 +3,8 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     Task<IEnumerable<string>> GetKnowledgeCollections();
-    Task<IEnumerable<KnowledgeRetrievalResult>> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options);
+    Task<IEnumerable<KnowledgeSearchResult>> SearchKnowledge(string collectionName, KnowledgeSearchOptions options);
     Task FeedKnowledge(string collectionName, KnowledgeCreationModel model);
-    Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);
+    Task<StringIdPagedItems<KnowledgeSearchResult>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);
     Task<bool> DeleteKnowledgeCollectionData(string collectionName, string id);
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs

@@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 public class KnowledgeCollectionData
 {
     public string Id { get; set; }
-    public string Question { get; set; }
-    public string Answer { get; set; }
+    public Dictionary<string, string> Data { get; set; } = new();
+    public double? Score { get; set; }
     public float[]? Vector { get; set; }
-}
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs renamed to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs

@@ -2,7 +2,7 @@
 
 namespace BotSharp.Abstraction.Knowledges.Models;
 
-public class KnowledgeRetrievalOptions
+public class KnowledgeSearchOptions
 {
     public string Text { get; set; } = string.Empty;
     public IEnumerable<string>? Fields { get; set; } = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs

@@ -1,12 +1,20 @@
 namespace BotSharp.Abstraction.Knowledges.Models;
 
-public class KnowledgeSearchResult
+public class KnowledgeSearchResult : KnowledgeCollectionData
 {
-    public Dictionary<string, string> Data { get; set; } = new();
-    public double Score { get; set; }
-    public float[]? Vector { get; set; }
-}
+    public KnowledgeSearchResult()
+    {
+        
+    }
 
-public class KnowledgeRetrievalResult : KnowledgeSearchResult
-{
+    public static KnowledgeSearchResult CopyFrom(KnowledgeCollectionData data)
+    {
+        return new KnowledgeSearchResult
+        {
+            Id = data.Id,
+            Data = data.Data,
+            Score = data.Score,
+            Vector = data.Vector
+        };
+    }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.VectorStorage;
 public interface IVectorDb
 {
     string Name { get; }
-
+    
     Task<IEnumerable<string>> GetCollections();
     Task<StringIdPagedItems<KnowledgeCollectionData>> GetCollectionData(string collectionName, KnowledgeFilter filter);
     Task CreateCollection(string collectionName, int dim);
     Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
-    Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector, IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
+    Task<IEnumerable<KnowledgeCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
     Task<bool> DeleteCollectionData(string collectionName, string id);
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -23,29 +23,29 @@ public async Task<IEnumerable<string>> GetKnowledgeCollections()
     }
 
     [HttpPost("/knowledge/{collection}/search")]
-    public async Task<IEnumerable<KnowledgeRetrivalViewModel>> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model)
+    public async Task<IEnumerable<KnowledgeSearchResultViewModel>> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request)
     {
-        var options = new KnowledgeRetrievalOptions
+        var options = new KnowledgeSearchOptions
         {
-            Text = model.Text,
-            Fields = model.Fields,
-            Limit = model.Limit ?? 5,
-            Confidence = model.Confidence ?? 0.5f,
-            WithVector = model.WithVector
+            Text = request.Text,
+            Fields = request.Fields,
+            Limit = request.Limit ?? 5,
+            Confidence = request.Confidence ?? 0.5f,
+            WithVector = request.WithVector
         };
 
         var results = await _knowledgeService.SearchKnowledge(collection, options);
-        return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList();
+        return results.Select(x => KnowledgeSearchResultViewModel.From(x)).ToList();
     }
 
     [HttpPost("/knowledge/{collection}/data")]
-    public async Task<StringIdPagedItems<KnowledgeCollectionDataViewModel>> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter)
+    public async Task<StringIdPagedItems<KnowledgeSearchResultViewModel>> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter)
     {
         var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter);
-        var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))?
-                               .ToList() ?? new List<KnowledgeCollectionDataViewModel>();
+        var items = data.Items?.Select(x => KnowledgeSearchResultViewModel.From(x))?
+                               .ToList() ?? new List<KnowledgeSearchResultViewModel>();
 
-        return new StringIdPagedItems<KnowledgeCollectionDataViewModel>
+        return new StringIdPagedItems<KnowledgeSearchResultViewModel>
         {
             Count = data.Count,
             NextId = data.NextId,

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs

@@ -0,0 +1,33 @@
+using BotSharp.Abstraction.Knowledges.Models;
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+
+public class KnowledgeSearchResultViewModel
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("data")]
+    public IDictionary<string, string> Data { get; set; }
+
+    [JsonPropertyName("score")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public double? Score { get; set; }
+
+    [JsonPropertyName("vector")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public float[]? Vector { get; set; }
+
+
+    public static KnowledgeSearchResultViewModel From(KnowledgeSearchResult result)
+    {
+        return new KnowledgeSearchResultViewModel
+        {
+            Id = result.Id,
+            Data = result.Data,
+            Score = result.Score,
+            Vector = result.Vector
+        };
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs

@@ -3,7 +3,7 @@
 
 namespace BotSharp.OpenAPI.ViewModels.Knowledges;
 
-public class SearchKnowledgeModel
+public class SearchKnowledgeRequest
 {
     [JsonPropertyName("text")]
     public string Text { get; set; } = string.Empty;