add vector data source

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeDocSource.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Knowledges.Enums;
+
+public static class KnowledgeDocSource
+{
+    public const string Api = "api";
+    public const string User = "user";
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeDocType.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Knowledges.Enums;
+
+public class KnowledgeDocType
+{
+    public const string File = "file";
+    public const string Http = "http";
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs

@@ -7,4 +7,5 @@ public static class KnowledgePayloadName
     public static string Answer = "answer";
     public static string Request = "request";
     public static string Response = "response";
+    public static string DataSource = "dataSource";
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Enums/VectorDataSource.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.VectorStorage.Enums;
+
+public static class VectorDataSource
+{
+    public const string Api = "api";
+    public const string User = "user";
+    public const string File = "file";
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs

@@ -1,7 +1,10 @@
+using BotSharp.Abstraction.VectorStorage.Enums;
+
 namespace BotSharp.Abstraction.VectorStorage.Models;
 
 public class VectorCreateModel
 {
     public string Text { get; set; }
+    public string DataSource { get; set; } = VectorDataSource.Api;
     public Dictionary<string, string>? Payload { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -10,4 +10,11 @@ public class VectorFilter : StringIdPagination
     /// </summary>
     [JsonPropertyName("search_pairs")]
     public IEnumerable<KeyValue>? SearchPairs { get; set; }
+
+
+    /// <summary>
+    /// Included payload keys
+    /// </summary>
+    [JsonPropertyName("included_payloads")]
+    public IEnumerable<string>? IncludedPayloads { get; set; }
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -1,11 +1,8 @@
-using Azure.Core;
-using BotSharp.Abstraction.Files.Constants;
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Graph.Models;
 using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
-using System.IO;
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -77,6 +74,7 @@ public async Task<bool> CreateVectorKnowledge([FromRoute] string collection, [Fr
         var create = new VectorCreateModel
         {
             Text = request.Text,
+            DataSource = request.DataSource,
             Payload = request.Payload
         };
 
@@ -91,6 +89,7 @@ public async Task<bool> UpdateVectorKnowledge([FromRoute] string collection, [Fr
         {
             Id = request.Id,
             Text = request.Text,
+            DataSource = request.DataSource,
             Payload = request.Payload
         };
 
@@ -106,24 +105,6 @@ public async Task<bool> DeleteVectorCollectionData([FromRoute] string collection
     #endregion
 
 
-    #region Graph
-    [HttpPost("/knowledge/graph/search")]
-    public async Task<GraphKnowledgeViewModel> SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request)
-    {
-        var options = new GraphSearchOptions
-        {
-            Method = request.Method
-        };
-
-        var result = await _knowledgeService.SearchGraphKnowledge(request.Query, options);
-        return new GraphKnowledgeViewModel
-        {
-            Result = result.Result
-        };
-    }
-    #endregion
-
-
     #region Document
     [HttpPost("/knowledge/document/{collection}/upload")]
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
@@ -181,6 +162,25 @@ public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collect
     #endregion
 
 
+
+    #region Graph
+    [HttpPost("/knowledge/graph/search")]
+    public async Task<GraphKnowledgeViewModel> SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request)
+    {
+        var options = new GraphSearchOptions
+        {
+            Method = request.Method
+        };
+
+        var result = await _knowledgeService.SearchGraphKnowledge(request.Query, options);
+        return new GraphKnowledgeViewModel
+        {
+            Result = result.Result
+        };
+    }
+    #endregion
+
+
     #region Common
     [HttpPost("/knowledge/vector/refresh-configs")]
     public async Task<string> RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request)

src/Infrastructure/BotSharp.OpenAPI/Using.cs

@@ -26,7 +26,8 @@
 global using BotSharp.Abstraction.Repositories.Filters;
 global using BotSharp.Abstraction.Files.Models;
 global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.VectorStorage.Enums;
 global using BotSharp.OpenAPI.ViewModels.Conversations;
 global using BotSharp.OpenAPI.ViewModels.Users;
 global using BotSharp.OpenAPI.ViewModels.Agents;
-global using BotSharp.OpenAPI.ViewModels.Files;
+global using BotSharp.OpenAPI.ViewModels.Files;

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs

@@ -7,6 +7,9 @@ public class VectorKnowledgeCreateRequest
     [JsonPropertyName("text")]
     public string Text { get; set; }
 
+    [JsonPropertyName("data_source")]
+    public string DataSource { get; set; } = VectorDataSource.Api;
+
     [JsonPropertyName("payload")]
     public Dictionary<string, string>? Payload { get; set; }
 }

src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs

@@ -37,7 +37,8 @@ private bool VerifyGetRequest(HttpRequest request)
         var regexes = new List<Regex>
         {
             new Regex(@"/conversation/(.*?)/message/(.*?)/(.*?)/file/(.*?)/(.*?)", RegexOptions.IgnoreCase),
-            new Regex(@"/user/avatar", RegexOptions.IgnoreCase)
+            new Regex(@"/user/avatar", RegexOptions.IgnoreCase),
+            new Regex(@"/knowledge/document/(.*?)/file/(.*?)", RegexOptions.IgnoreCase)
         };
 
         return request.Method.IsEqualTo("GET") && regexes.Any(x => x.IsMatch(request.Path.Value ?? string.Empty));

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.VectorStorage.Enums;
 
 namespace BotSharp.Plugin.KnowledgeBase.Services;
 
@@ -119,7 +120,10 @@ public async Task<bool> CreateVectorCollectionData(string collectionName, Vector
 
             var db = GetVectorDb();
             var guid = Guid.NewGuid();
-            return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
+            var payload = create.Payload ?? new();
+            payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(create.DataSource) ? create.DataSource : VectorDataSource.Api;
+
+            return await db.Upsert(collectionName, guid, vector, create.Text, payload);
         }
         catch (Exception ex)
         {
@@ -146,7 +150,10 @@ public async Task<bool> UpdateVectorCollectionData(string collectionName, Vector
 
             var textEmbedding = GetTextEmbedding(collectionName);
             var vector = await textEmbedding.GetVectorAsync(update.Text);
-            return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
+            var payload = update.Payload ?? new();
+            payload[KnowledgePayloadName.DataSource] = !string.IsNullOrWhiteSpace(update.DataSource) ? update.DataSource : VectorDataSource.Api;
+
+            return await db.Upsert(collectionName, guid, vector, update.Text, payload);
         }
         catch (Exception ex)
         {

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Utilities;
 using BotSharp.Abstraction.VectorStorage.Models;
+using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Logging;
 using Qdrant.Client;
 using Qdrant.Client.Grpc;
@@ -112,10 +113,25 @@ public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionDa
             };
         }
 
+        // Build payload selector
+        WithPayloadSelector? payloadSelector = null;
+        if (!filter.IncludedPayloads.IsNullOrEmpty())
+        {
+            payloadSelector = new WithPayloadSelector
+            { 
+                Enable = true,
+                Include = new PayloadIncludeSelector
+                {
+                    Fields = { filter.IncludedPayloads.ToArray() }
+                }
+            };
+        }
+
         var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
         var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, 
             offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
             filter: queryFilter,
+            payloadSelector: payloadSelector,
             vectorsSelector: filter.WithVector);
 
         var points = response?.Result?.Select(x => new VectorCollectionData