diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index 8aeba2d2a..018f6086c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -23,6 +23,8 @@ public class CleanConversationSetting public int BatchSize { get; set; } public int MessageLimit { get; set; } public int BufferHours { get; set; } + public int LogRetentionDays { get; set; } + public int LogBatchSize { get; set; } = 2000; public IEnumerable ExcludeAgentIds { get; set; } = new List(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index cb00cddfd..81657b651 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -198,6 +198,11 @@ Task> GetConversationStateLogs(str => throw new NotImplementedException(); #endregion + #region Log Cleanup + Task DeleteOldConversationLogs(int retentionDays, int batchSize) + => throw new NotImplementedException(); + #endregion + #region Instruction Log Task SaveInstructionLogs(IEnumerable logs) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index f17c767b4..4e51966d1 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -178,6 +178,15 @@ public async Task> GetConversation } #endregion + #region Log Cleanup + public async Task DeleteOldConversationLogs(int retentionDays, int batchSize = 2000) + { + // For file repository, we might not need to implement this fully or we can just return 0 + // as it's typically used for local dev. Implementing it would require iterating all conversations. + return await Task.FromResult(0); + } + #endregion + #region Instruction Log public async Task SaveInstructionLogs(IEnumerable logs) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs deleted file mode 100644 index 4e76dc069..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ /dev/null @@ -1,77 +0,0 @@ -using Microsoft.Extensions.Hosting; - -namespace BotSharp.OpenAPI.BackgroundServices -{ - public class ConversationTimeoutService : BackgroundService - { - private readonly IServiceProvider _services; - private readonly ILogger _logger; - - public ConversationTimeoutService(IServiceProvider services, ILogger logger) - { - _services = services; - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Conversation Timeout Service is running..."); - - _ = Task.Run(async () => - { - await DoWork(stoppingToken); - }); - } - - private async Task DoWork(CancellationToken stoppingToken) - { - _logger.LogInformation("Conversation Timeout Service is doing work..."); - - try - { - while (true) - { - stoppingToken.ThrowIfCancellationRequested(); - var delay = Task.Delay(TimeSpan.FromHours(1)); - try - { - await CleanIdleConversationsAsync(); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error occurred closing conversations."); - } - await delay; - } - } - catch (OperationCanceledException) { } - } - - public override async Task StopAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Conversation Timeout Service is stopping."); - await base.StopAsync(stoppingToken); - } - - private async Task CleanIdleConversationsAsync() - { - using var scope = _services.CreateScope(); - var settings = scope.ServiceProvider.GetRequiredService(); - var cleanSetting = settings.CleanSetting; - - if (cleanSetting == null || !cleanSetting.Enable) return; - - var conversationService = scope.ServiceProvider.GetRequiredService(); - var batchSize = cleanSetting.BatchSize; - var limit = cleanSetting.MessageLimit; - var bufferHours = cleanSetting.BufferHours; - var excludeAgentIds = cleanSetting.ExcludeAgentIds ?? new List(); - var conversationIds = await conversationService.GetIdleConversations(batchSize, limit, bufferHours, excludeAgentIds); - - if (!conversationIds.IsNullOrEmpty()) - { - await conversationService.DeleteConversations(conversationIds); - } - } - } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs index fff51cace..6d288bff5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs @@ -1,7 +1,8 @@ using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Core.MCP.Settings; using BotSharp.Core.Users.Services; -using BotSharp.OpenAPI.BackgroundServices; +using BotSharp.OpenAPI.Hooks; +using BotSharp.OpenAPI.RuleTriggers; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Authentication.JwtBearer; @@ -14,6 +15,10 @@ using Microsoft.Net.Http.Headers; using Microsoft.OpenApi; using System.Text.Json.Serialization; +using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Rules; + + #if NET8_0 using Microsoft.OpenApi.Models; #endif @@ -36,7 +41,8 @@ public static IServiceCollection AddBotSharpOpenAPI(this IServiceCollection serv bool enableValidation) { services.AddScoped(); - services.AddHostedService(); + + services.AddCrontabServices(); // Add bearer authentication var schema = "MIXED_SCHEME"; @@ -223,6 +229,18 @@ private static async Task OnTicketReceivedContext(TicketReceivedContext context) } } + public static IServiceCollection AddCrontabServices(this IServiceCollection services) + { + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + return services; + } + /// /// Use Swagger/OpenAPI /// diff --git a/src/Infrastructure/BotSharp.OpenAPI/Hooks/ConversationLogCleanupCrontabHook.cs b/src/Infrastructure/BotSharp.OpenAPI/Hooks/ConversationLogCleanupCrontabHook.cs new file mode 100644 index 000000000..b9cba7191 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Hooks/ConversationLogCleanupCrontabHook.cs @@ -0,0 +1,69 @@ +using BotSharp.Abstraction.Conversations.Settings; +using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Crontab.Models; +using BotSharp.Abstraction.Repositories; +using BotSharp.OpenAPI.RuleTriggers; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Threading.Tasks; + +namespace BotSharp.OpenAPI.Hooks +{ + public class ConversationLogCleanupCrontabHook : ICrontabHook + { + private readonly ConversationSetting _settings; + private readonly IBotSharpRepository _db; + private readonly ILogger _logger; + private readonly IHostApplicationLifetime _appLifetime; + + public string[]? Triggers => new[] { nameof(ConversationLogCleanupRuleTrigger) }; + + public ConversationLogCleanupCrontabHook( + ConversationSetting settings, + IBotSharpRepository db, + ILogger logger, + IHostApplicationLifetime appLifetime) + { + _settings = settings; + _db = db; + _logger = logger; + _appLifetime = appLifetime; + } + + public async Task OnCronTriggered(CrontabItem item) + { + var cleanSetting = _settings.CleanSetting; + + if (cleanSetting == null || !cleanSetting.Enable || cleanSetting.LogRetentionDays <= 0) return; + + int totalDeleted = 0; + var cancellationToken = _appLifetime.ApplicationStopping; + + while (!cancellationToken.IsCancellationRequested) + { + var deletedCount = await _db.DeleteOldConversationLogs(cleanSetting.LogRetentionDays, cleanSetting.LogBatchSize); + if (deletedCount == 0) break; + + totalDeleted += deletedCount; + _logger.LogInformation($"Cleaned {deletedCount} conversation logs older than {cleanSetting.LogRetentionDays} days in this batch."); + + try + { + // Sleep slightly to yield database resources, will throw TaskCanceledException on shutdown + await Task.Delay(1000, cancellationToken); + } + catch (OperationCanceledException) + { + _logger.LogWarning("Conversation log cleanup was interrupted due to application shutdown."); + break; + } + } + + if (totalDeleted > 0) + { + _logger.LogInformation($"Successfully cleaned a total of {totalDeleted} conversation logs older than {cleanSetting.LogRetentionDays} days."); + } + } + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Hooks/IdleConversationCleanupCrontabHook.cs b/src/Infrastructure/BotSharp.OpenAPI/Hooks/IdleConversationCleanupCrontabHook.cs new file mode 100644 index 000000000..52f11d70e --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Hooks/IdleConversationCleanupCrontabHook.cs @@ -0,0 +1,53 @@ +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Settings; +using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Crontab.Models; +using BotSharp.OpenAPI.RuleTriggers; +using Microsoft.Extensions.Logging; + +namespace BotSharp.OpenAPI.Hooks +{ + public class IdleConversationCleanupCrontabHook : ICrontabHook + { + private readonly ConversationSetting _settings; + private readonly IConversationService _conversationService; + private readonly ILogger _logger; + + public string[]? Triggers => new[] { nameof(IdleConversationCleanupRuleTrigger) }; + + public IdleConversationCleanupCrontabHook( + ConversationSetting settings, + IConversationService conversationService, + ILogger logger) + { + _settings = settings; + _conversationService = conversationService; + _logger = logger; + } + + public async Task OnCronTriggered(CrontabItem item) + { + var cleanSetting = _settings.CleanSetting; + + if (cleanSetting == null || !cleanSetting.Enable) return; + + try + { + var batchSize = cleanSetting.BatchSize; + var limit = cleanSetting.MessageLimit; + var bufferHours = cleanSetting.BufferHours; + var excludeAgentIds = cleanSetting.ExcludeAgentIds ?? new List(); + var conversationIds = await _conversationService.GetIdleConversations(batchSize, limit, bufferHours, excludeAgentIds); + + if (!conversationIds.IsNullOrEmpty()) + { + await _conversationService.DeleteConversations(conversationIds); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error occurred closing conversations."); + } + } + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/ConversationLogCleanupRuleTrigger.cs b/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/ConversationLogCleanupRuleTrigger.cs new file mode 100644 index 000000000..d4b200695 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/ConversationLogCleanupRuleTrigger.cs @@ -0,0 +1,26 @@ +using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Crontab.Models; +using BotSharp.Abstraction.Rules; + +namespace BotSharp.OpenAPI.RuleTriggers +{ + public class ConversationLogCleanupRuleTrigger : IRuleTrigger, ICrontabSource + { + public string Channel => ConversationChannel.Crontab; + public string Name => nameof(ConversationLogCleanupRuleTrigger); + public string EntityType { get; set; } = string.Empty; + public string EntityId { get; set; } = string.Empty; + + public CrontabItem GetCrontabItem() + { + return new CrontabItem + { + Title = nameof(ConversationLogCleanupRuleTrigger), + Description = "Clean up old conversation logs daily", + Cron = "0 6 * * *", // Run at 6:00 AM UTC (Midnight Chicago Standard Time) + TriggerType = CronTabItemTriggerType.MessageQueue + }; + } + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/IdleConversationCleanupRuleTrigger.cs b/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/IdleConversationCleanupRuleTrigger.cs new file mode 100644 index 000000000..f965d5671 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/IdleConversationCleanupRuleTrigger.cs @@ -0,0 +1,26 @@ +using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Crontab.Models; +using BotSharp.Abstraction.Rules; + +namespace BotSharp.OpenAPI.RuleTriggers +{ + public class IdleConversationCleanupRuleTrigger : IRuleTrigger, ICrontabSource + { + public string Channel => ConversationChannel.Crontab; + public string Name => nameof(IdleConversationCleanupRuleTrigger); + public string EntityType { get; set; } = string.Empty; + public string EntityId { get; set; } = string.Empty; + + public CrontabItem GetCrontabItem() + { + return new CrontabItem + { + Title = nameof(IdleConversationCleanupRuleTrigger), + Description = "Clean up idle conversations hourly", + Cron = "0 * * * *", + TriggerType = CronTabItemTriggerType.MessageQueue + }; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index f851d5625..2444f4bd0 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -139,6 +139,37 @@ public async Task> GetConversation } #endregion + #region Log Cleanup + public async Task DeleteOldConversationLogs(int retentionDays, int batchSize) + { + if (retentionDays <= 0) return 0; + var threshold = DateTime.UtcNow.AddDays(-retentionDays); + + var contentLogFilter = Builders.Filter.Lt(x => x.CreatedTime, threshold); + var stateLogFilter = Builders.Filter.Lt(x => x.CreatedTime, threshold); + + var contentDocsToDelete = await _dc.ContentLogs.Find(contentLogFilter).Limit(batchSize).Project(x => x.Id).ToListAsync(); + long contentDeletedCount = 0; + if (contentDocsToDelete.Any()) + { + var deleteFilter = Builders.Filter.In(x => x.Id, contentDocsToDelete); + var contentDeleted = await _dc.ContentLogs.DeleteManyAsync(deleteFilter); + contentDeletedCount = contentDeleted.DeletedCount; + } + + var stateDocsToDelete = await _dc.StateLogs.Find(stateLogFilter).Limit(batchSize).Project(x => x.Id).ToListAsync(); + long stateDeletedCount = 0; + if (stateDocsToDelete.Any()) + { + var deleteFilter = Builders.Filter.In(x => x.Id, stateDocsToDelete); + var stateDeleted = await _dc.StateLogs.DeleteManyAsync(deleteFilter); + stateDeletedCount = stateDeleted.DeletedCount; + } + + return (int)(contentDeletedCount + stateDeletedCount); + } + #endregion + #region Instruction Log public async Task SaveInstructionLogs(IEnumerable logs) { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 9cca46170..42a82f33c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -707,6 +707,8 @@ "BatchSize": 50, "MessageLimit": 2, "BufferHours": 12, + "LogRetentionDays": 30, + "LogBatchSize": 2000, "ExcludeAgentIds": [] }, "RateLimit": { diff --git a/tests/BotSharp.LLM.Tests/appsettings.json b/tests/BotSharp.LLM.Tests/appsettings.json index 41ca7076d..7e916a14a 100644 --- a/tests/BotSharp.LLM.Tests/appsettings.json +++ b/tests/BotSharp.LLM.Tests/appsettings.json @@ -201,6 +201,8 @@ "BatchSize": 50, "MessageLimit": 2, "BufferHours": 12, + "LogRetentionDays": 30, + "LogBatchSize": 2000, "ExcludeAgentIds": [] }, "RateLimit": {