Skip to content

1. split knowledge menu #598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ public class PluginMenuDef
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<PluginMenuDef>? SubMenu { get; set; }

public PluginMenuDef(string lable, string? link = null, string? icon = null, int weight = 0)
public PluginMenuDef(string label, string? link = null, string? icon = null, int weight = 0)
{
Label = lable;
Label = label;
Link = link;
Icon = icon;
Weight = weight;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public interface IVectorDb
Task<IEnumerable<string>> GetCollections();
Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
Task CreateCollection(string collectionName, int dim);
Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
Task<bool> DeleteCollectionData(string collectionName, string id);
Task<bool> DeleteCollectionData(string collectionName, Guid id);
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,8 @@ public async Task<bool> Execute(RoleDialogModel message)
var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
await vectorDb.CreateCollection(collectionName, vector[0].Length);

var id = Guid.NewGuid().ToString();
var result = await vectorDb.Upsert(collectionName, id, vector[0],
args.Question,
var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0],
args.Question,
new Dictionary<string, string>
{
{ KnowledgePayloadName.Answer, args.Answer }
Expand Down
11 changes: 8 additions & 3 deletions src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
});

var a = config["KnowledgeBase"];

services.AddScoped<ITextChopper, TextChopperService>();
services.AddScoped<IKnowledgeService, KnowledgeService>();
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
Expand All @@ -32,7 +30,14 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
public bool AttachMenu(List<PluginMenuDef> menu)
{
var section = menu.First(x => x.Label == "Apps");
menu.Add(new PluginMenuDef("Knowledge Base", link: "/page/knowledge-base", icon: "bx bx-book-open", weight: section.Weight + 1));
menu.Add(new PluginMenuDef("Knowledge Base", icon: "bx bx-book-open", weight: section.Weight + 1)
{
SubMenu = new List<PluginMenuDef>
{
new PluginMenuDef("Vector", link: "page/knowledge-base/vector"),
new PluginMenuDef("Graph", link: "page/knowledge-base/graph")
}
});
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,19 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
return await Task.FromResult(results);
}

public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
_vectors[collectionName].Add(new VecRecord
{
Id = id,
Id = id.ToString(),
Vector = vector,
Text = text
});

return true;
}

public async Task<bool> DeleteCollectionData(string collectionName, string id)
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
return await Task.FromResult(false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationMo
foreach (var line in lines)
{
var vec = await textEmbedding.GetVectorAsync(line);
var id = Guid.NewGuid().ToString();
await db.Upsert(collectionName, id, vec, line);
await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
index++;
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,13 @@ public async Task<bool> DeleteVectorCollectionData(string collectionName, string
{
try
{
if (!Guid.TryParse(id, out var guid))
{
return false;
}

var db = GetVectorDb();
return await db.DeleteCollectionData(collectionName, id);
return await db.DeleteCollectionData(collectionName, guid);
}
catch (Exception ex)
{
Expand Down
4 changes: 2 additions & 2 deletions src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, flo
throw new NotImplementedException();
}

public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
throw new NotImplementedException();
}

public Task<bool> DeleteCollectionData(string collectionName, string id)
public Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
throw new NotImplementedException();
}
Expand Down
19 changes: 9 additions & 10 deletions src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,14 @@ public async Task CreateCollection(string collectionName, int dim)
}
}

public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
// Insert vectors
var point = new PointStruct()
{
Id = new PointId()
{
Uuid = id
Uuid = id.ToString()
},
Vectors = vector,
Payload =
Expand Down Expand Up @@ -137,7 +137,11 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
return results;
}

var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
var points = await client.SearchAsync(collectionName,
vector,
limit: (ulong)limit,
scoreThreshold: confidence,
vectorsSelector: new WithVectorsSelector { Enable = withVector });

var pickFields = fields != null;
foreach (var point in points)
Expand Down Expand Up @@ -174,15 +178,10 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
return results;
}

public async Task<bool> DeleteCollectionData(string collectionName, string id)
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
if (!Guid.TryParse(id, out var guid))
{
return false;
}

var client = GetClient();
var result = await client.DeleteAsync(collectionName, guid);
var result = await client.DeleteAsync(collectionName, id);
return result.Status == UpdateStatus.Completed;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using BotSharp.Abstraction.VectorStorage;
using BotSharp.Abstraction.VectorStorage.Models;
using Microsoft.SemanticKernel.Memory;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;

Expand Down Expand Up @@ -62,21 +63,21 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
return resultTexts;
}

public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload)
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)
{
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
return true;
}

public async Task<bool> DeleteCollectionData(string collectionName, string id)
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);

if (exist)
{
await _memoryStore.RemoveAsync(collectionName, id);
await _memoryStore.RemoveAsync(collectionName, id.ToString());
return true;
}
return false;
Expand Down