add global stats

src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/CacheType.cs

@@ -1,14 +1,7 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+namespace BotSharp.Abstraction.Infrastructures.Enums;
 
-namespace BotSharp.Abstraction.Infrastructures.Enums
+public enum CacheType
 {
-    public enum CacheType
-    {
-        MemoryCache,
-        RedisCache
-    }
+    MemoryCache = 1,
+    RedisCache = 2
 }

src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheSettings.cs

@@ -3,6 +3,6 @@ namespace BotSharp.Abstraction.Infrastructures;
 public class SharpCacheSettings
 {
     public bool Enabled { get; set; } = true;
-    public CacheType CacheType { get; set; } = Enums.CacheType.MemoryCache;
+    public CacheType CacheType { get; set; } = CacheType.MemoryCache;
     public string Prefix { get; set; } = "cache";    
 }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -3,6 +3,7 @@
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Roles.Models;
 using BotSharp.Abstraction.Shared;
+using BotSharp.Abstraction.Statistics.Models;
 using BotSharp.Abstraction.Tasks.Models;
 using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Enums;
@@ -119,7 +120,9 @@ public interface IBotSharpRepository : IHaveServiceProvider
     #endregion
 
     #region Statistics
-    void IncrementConversationCount();
+    BotSharpStats? GetGlobalStats(string category, string group, DateTime recordTime) => throw new NotImplementedException();
+    bool SaveGlobalStats(BotSharpStats body) => throw new NotImplementedException();
+
     #endregion
 
     #region Translation

src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatCategory.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Statistics.Enums;
+
+public static class StatCategory
+{
+    public static string LlmCost = "llm-cost";
+    public static string AgentCall = "agent-call";
+}

src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs

@@ -0,0 +1,34 @@
+namespace BotSharp.Abstraction.Statistics.Models;
+
+public class BotSharpStats
+{
+    [JsonPropertyName("category")]
+    public string Category { get; set; } = null!;
+
+    [JsonPropertyName("group")]
+    public string Group { get; set; } = null!;
+
+    [JsonPropertyName("data")]
+    public IDictionary<string, object> Data { get; set; } = new Dictionary<string, object>();
+
+    private DateTime innerRecordTime;
+
+    [JsonPropertyName("record_time")]
+    public DateTime RecordTime
+    {
+        get
+        {
+            return innerRecordTime;
+        }
+        set
+        {
+            var date = new DateTime(value.Year, value.Month, value.Day, value.Hour, 0, 0);
+            innerRecordTime = DateTime.SpecifyKind(date, DateTimeKind.Utc);
+        }
+    }
+
+    public override string ToString()
+    {
+        return $"{Category}-{Group}: {Data?.Count ?? 0} ({RecordTime})";
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatService.cs

@@ -0,0 +1,9 @@
+using BotSharp.Abstraction.Statistics.Models;
+
+namespace BotSharp.Abstraction.Statistics.Services;
+
+public interface IBotSharpStatService
+{
+    bool UpdateLlmCost(BotSharpStats stats);
+    bool UpdateAgentCall(BotSharpStats stats);
+}

src/Infrastructure/BotSharp.Abstraction/Statistics/Settings/StatisticsSettings.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
+namespace BotSharp.Abstraction.Statistics.Settings;
 
-namespace BotSharp.Abstraction.Statistics.Settings
+public class StatisticsSettings
 {
-    public class StatisticsSettings
-    {
-        public string DataDir { get; set; }
-    }
+    public bool Enabled { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Utilities/MathExt.cs

@@ -11,4 +11,9 @@ public static long Max(long a, long b, long c)
     {
         return Math.Max(Math.Max(a, b), c);
     }
+
+    public static decimal Round(decimal value, MidpointRounding rouding = MidpointRounding.AwayFromZero)
+    {
+        return Math.Round(value, rouding);
+    }
 }

src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs

@@ -32,6 +32,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<ILlmProviderService, LlmProviderService>();
         services.AddScoped<IAgentService, AgentService>();
         services.AddScoped<IAgentHook, BasicAgentHook>();
+        services.AddScoped<IBotSharpStatService, BotSharpStatService>();
 
         services.AddScoped(provider =>
         {

src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs

@@ -1,7 +1,6 @@
 using BotSharp.Abstraction.Conversations.Enums;
 using BotSharp.Abstraction.MLTasks;
 using System.Diagnostics;
-using System.Drawing;
 
 namespace BotSharp.Core.Conversations.Services;
 
@@ -42,8 +41,10 @@ public void AddToken(TokenStatsModel stats)
         var settingsService = _services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(stats.Provider, _model);
 
-        _promptCost += stats.PromptCount / 1000f * settings.PromptCost;
-        _completionCost += stats.CompletionCount / 1000f * settings.CompletionCost;
+        var deltaPromptCost = stats.PromptCount / 1000f * settings.PromptCost;
+        var deltaCompletionCost = stats.CompletionCount / 1000 * settings.CompletionCost;
+        _promptCost += deltaPromptCost;
+        _completionCost += deltaCompletionCost;
 
         // Accumulated Token
         var stat = _services.GetRequiredService<IConversationStateService>();
@@ -56,6 +57,23 @@ public void AddToken(TokenStatsModel stats)
         var total_cost = float.Parse(stat.GetState("llm_total_cost", "0"));
         total_cost += Cost;
         stat.SetState("llm_total_cost", total_cost, isNeedVersion: false, source: StateSource.Application);
+
+
+        var globalStats = _services.GetRequiredService<IBotSharpStatService>();
+        var body = new BotSharpStats
+        {
+            Category = StatCategory.LlmCost,
+            Group = $"Provider: {stats.Provider} | Model: {stats.Model}",
+            Data = new Dictionary<string, object>
+            {
+                { "prompt_token_count_total", stats.PromptCount },
+                { "completion_token_count_total", stats.CompletionCount },
+                { "prompt_cost_total", deltaPromptCost },
+                { "completion_cost_total", deltaCompletionCost }
+            },
+            RecordTime = DateTime.UtcNow
+        };
+        globalStats.UpdateLlmCost(body);
     }
 
     public void PrintStatistics()

src/Infrastructure/BotSharp.Core/Infrastructures/DistributedLocker.cs

@@ -9,7 +9,9 @@ public class DistributedLocker : IDistributedLocker
     private readonly IServiceProvider _services;
     private readonly ILogger _logger;
 
-    public DistributedLocker(IServiceProvider services, ILogger<DistributedLocker> logger)
+    public DistributedLocker(
+        IServiceProvider services,
+        ILogger<DistributedLocker> logger)
     {
         _services = services;
         _logger = logger;
@@ -46,6 +48,13 @@ public bool Lock(string resource, Action action, int timeoutInSeconds = 30)
         var timeout = TimeSpan.FromSeconds(timeoutInSeconds);
 
         var redis = _services.GetRequiredService<IConnectionMultiplexer>();
+        if (redis == null)
+        {
+            _logger.LogWarning($"The Redis server is experiencing issues and is not functioning as expected.");
+            action();
+            return false;
+        }
+
         var @lock = new RedisDistributedLock(resource, redis.GetDatabase());
         using (var handle = @lock.TryAcquire(timeout))
         {

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs

@@ -73,7 +73,7 @@ public void UpdateAgent(Agent agent, AgentField field)
                     break;
             }
 
-            _agents = [];
+            ResetInnerAgents();
         }
 
         #region Update Agent Fields
@@ -541,7 +541,7 @@ public void BulkInsertAgents(List<Agent> agents)
                 }
             }
 
-            ResetLocalAgents();
+            ResetInnerAgents();
         }
 
         public void BulkInsertUserAgents(List<UserAgent> userAgents)
@@ -574,7 +574,7 @@ public void BulkInsertUserAgents(List<UserAgent> userAgents)
                 Thread.Sleep(50);
             }
 
-            ResetLocalAgents();
+            ResetInnerAgents();
         }
 
         public bool DeleteAgents()
@@ -629,7 +629,7 @@ public bool DeleteAgent(string agentId)
 
                 // Delete agent folder
                 Directory.Delete(agentDir, true);
-                ResetLocalAgents();
+                ResetInnerAgents();
                 return true;
             }
             catch
@@ -638,7 +638,7 @@ public bool DeleteAgent(string agentId)
             }
         }
 
-        private void ResetLocalAgents()
+        private void ResetInnerAgents()
         {
             _agents = [];
             _userAgents = [];

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs

@@ -1,57 +1,68 @@
-using BotSharp.Abstraction.Statistics.Model;
 using System.IO;
 
-namespace BotSharp.Core.Repository
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
 {
-    public partial class FileRepository
+    public BotSharpStats? GetGlobalStats(string category, string group, DateTime recordTime)
+    {
+        var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
+        var dir = Path.Combine(baseDir, category, recordTime.Year.ToString(), recordTime.Month.ToString("D2"));
+        if (!Directory.Exists(dir)) return null;
+
+        var file = Directory.GetFiles(dir).FirstOrDefault(x => Path.GetFileName(x) == STATS_FILE);
+        if (file == null) return null;
+
+        var text = File.ReadAllText(file);
+        var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
+        var found = list?.FirstOrDefault(x => x.Category.IsEqualTo(category)
+                                            && x.Group.IsEqualTo(group)
+                                            && x.RecordTime == recordTime);
+        return found;
+    }
+
+    public bool SaveGlobalStats(BotSharpStats body)
     {
-        public void IncrementConversationCount()
+        var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
+        var dir = Path.Combine(baseDir, body.Category, body.RecordTime.Year.ToString(), body.RecordTime.Month.ToString("D2"));
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
+
+        var file = Path.Combine(dir, STATS_FILE);
+        if (!File.Exists(file))
+        {
+            var list = new List<BotSharpStats> { body };
+            File.WriteAllText(file, JsonSerializer.Serialize(list, _options));
+        }
+        else
         {
-            var statsFileDirectory = FindCurrentStatsDirectory();
-            if (statsFileDirectory == null)
+            var text = File.ReadAllText(file);
+            var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
+            var found = list?.FirstOrDefault(x => x.Category.IsEqualTo(body.Category)
+                                                && x.Group.IsEqualTo(body.Group)
+                                                && x.RecordTime == body.RecordTime);
+
+            if (found != null)
             {
-                statsFileDirectory = CreateStatsFileDirectory();
+                found.Category = body.Category;
+                found.Group = body.Group;
+                found.Data = body.Data;
+                found.RecordTime = body.RecordTime;
             }
-            var fileName = GenerateStatsFileName();
-            var statsFile = Path.Combine(statsFileDirectory, fileName);
-            if (!File.Exists(statsFile))
+            else if (list != null)
             {
-                File.WriteAllText(statsFile, JsonSerializer.Serialize(new Statistics()
-                {
-                    Id = Guid.NewGuid().ToString(),
-                    UpdatedDateTime = DateTime.UtcNow
-                }, _options));
+                list.Add(body);
             }
-            var json = File.ReadAllText(statsFile);
-            var stats = JsonSerializer.Deserialize<Statistics>(json, _options);
-            stats.ConversationCount += 1;
-            stats.UpdatedDateTime = DateTime.UtcNow;
-            File.WriteAllText(statsFile, JsonSerializer.Serialize(stats, _options));
-        }
-        public string? CreateStatsFileDirectory()
-        {
-            var dir = GenerateStatsDirectoryName();
-            if (!Directory.Exists(dir))
+            else if (list == null)
             {
-                Directory.CreateDirectory(dir);
+                list = new List<BotSharpStats> { body };
             }
-            return dir;
-        }
-        private string? FindCurrentStatsDirectory()
-        {
-            var dir = GenerateStatsDirectoryName();
-            if (!Directory.Exists(dir)) return null;
 
-            return dir;
-        }
-        private string GenerateStatsDirectoryName()
-        {
-            return Path.Combine(_dbSettings.FileRepository, _statisticsSetting.DataDir, DateTime.UtcNow.Year.ToString(), DateTime.UtcNow.ToString("MM"));
-        }
-        private string GenerateStatsFileName()
-        {
-            var fileName = DateTime.UtcNow.ToString("MMdd");
-            return $"{fileName}-{STATS_FILE}";
+            File.WriteAllText(file, JsonSerializer.Serialize(list, _options));
         }
+
+        return true;
     }
 }