From bf9536d4c39460cad2d0ed72e703772b6dddfc19 Mon Sep 17 00:00:00 2001 From: hhvrc Date: Wed, 4 Dec 2024 22:19:44 +0100 Subject: [PATCH 1/5] Create ClearOldShockerControlLogs.cs --- Cron/Jobs/ClearOldShockerControlLogs.cs | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Cron/Jobs/ClearOldShockerControlLogs.cs diff --git a/Cron/Jobs/ClearOldShockerControlLogs.cs b/Cron/Jobs/ClearOldShockerControlLogs.cs new file mode 100644 index 00000000..b04f7448 --- /dev/null +++ b/Cron/Jobs/ClearOldShockerControlLogs.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore; +using OpenShock.Common; +using OpenShock.Common.Constants; +using OpenShock.Common.OpenShockDb; +using OpenShock.Cron.Attributes; + +namespace OpenShock.Cron.Jobs; + +/// +/// Deletes shocker control logs older than 9 months +/// +[CronJob("0 0 * * *")] // Every day at midnight (https://crontab.guru/) +public sealed class ClearOldShockerControlLogs +{ + private readonly OpenShockContext _db; + private readonly ILogger _logger; + + /// + /// DI constructor + /// + /// + /// + public ClearOldShockerControlLogs(OpenShockContext db, ILogger logger) + { + _db = db; + _logger = logger; + } + + public async Task Execute() + { + var earliestCreatedAtUtc = DateTime.UtcNow - Duration.PasswordResetRequestLifetime; + + // Run the delete query + int nDeleted = await _db.ShockerControlLogs + .Where(x => x.CreatedOn < earliestCreatedAtUtc) + .ExecuteDeleteAsync(); + + _logger.LogInformation("Deleted {deletedCount} shocker control logs since {earliestCreatedOnUtc}", nDeleted, earliestCreatedAtUtc); + } +} \ No newline at end of file From 8be109ade3c1b26708670c7c6bca1ec55759477c Mon Sep 17 00:00:00 2001 From: hhvrc Date: Wed, 5 Feb 2025 20:36:27 +0100 Subject: [PATCH 2/5] Make job also limit to one million logs --- Common/Constants/Constants.cs | 1 + Cron/Jobs/ClearOldShockerControlLogs.cs | 40 ++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/Common/Constants/Constants.cs b/Common/Constants/Constants.cs index 2d5d94ce..d6687f25 100644 --- a/Common/Constants/Constants.cs +++ b/Common/Constants/Constants.cs @@ -3,6 +3,7 @@ public static class Duration { public static readonly TimeSpan AuditRetentionTime = TimeSpan.FromDays(90); + public static readonly TimeSpan ShockerControlLogRetentionTime = TimeSpan.FromDays(365); public static readonly TimeSpan PasswordResetRequestLifetime = TimeSpan.FromHours(1); diff --git a/Cron/Jobs/ClearOldShockerControlLogs.cs b/Cron/Jobs/ClearOldShockerControlLogs.cs index b04f7448..7e265575 100644 --- a/Cron/Jobs/ClearOldShockerControlLogs.cs +++ b/Cron/Jobs/ClearOldShockerControlLogs.cs @@ -7,13 +7,14 @@ namespace OpenShock.Cron.Jobs; /// -/// Deletes shocker control logs older than 9 months +/// Deletes shocker control logs older than the retention period and enforces a maximum log count /// [CronJob("0 0 * * *")] // Every day at midnight (https://crontab.guru/) public sealed class ClearOldShockerControlLogs { private readonly OpenShockContext _db; private readonly ILogger _logger; + private const int MaxLogCount = 1_000_000; /// /// DI constructor @@ -28,13 +29,38 @@ public ClearOldShockerControlLogs(OpenShockContext db, ILogger x.CreatedOn < earliestCreatedAtUtc) - .ExecuteDeleteAsync(); + // Delete logs older than the retention threshold. + var deletedByAge = await _db.ShockerControlLogs + .Where(log => log.CreatedOn < retentionThreshold) + .ExecuteDeleteAsync(); - _logger.LogInformation("Deleted {deletedCount} shocker control logs since {earliestCreatedOnUtc}", nDeleted, earliestCreatedAtUtc); + _logger.LogInformation("Deleted {deletedCount} shocker control logs older than {retentionThreshold:O}.", deletedByAge, retentionThreshold); + + // Check the remaining number of logs to ensure it does not exceed the maximum allowed. + var remainingLogsCount = await _db.ShockerControlLogs.CountAsync(); + + if (remainingLogsCount > MaxLogCount) + { + _logger.LogInformation("Log count exceeds the maximum limit: {RemainingLogsCount} logs present, but the limit is {MaxLogCount}.", remainingLogsCount, MaxLogCount); + + // Identify the timestamp of the oldest log that should be retained to meet the log count limit. + var oldestLogToKeep = await _db.ShockerControlLogs + .OrderByDescending(log => log.CreatedOn) + .Select(log => log.CreatedOn) + .Skip(MaxLogCount) + .FirstAsync(); + + _logger.LogInformation("Preparing to delete logs created before {OldestLogToKeep:O} to enforce the log count limit.", oldestLogToKeep); + + // Delete logs that were created before the identified cutoff date. + var deletedByCount = await _db.ShockerControlLogs + .Where(log => log.CreatedOn < oldestLogToKeep) + .ExecuteDeleteAsync(); + + _logger.LogInformation("Deleted {DeletedByCount} additional logs older than {OldestLogToKeep:O}.", deletedByCount, oldestLogToKeep); + } } } \ No newline at end of file From bf2c9ee36b20458d266ea2ce98b34761cf7aedbe Mon Sep 17 00:00:00 2001 From: hhvrc Date: Wed, 5 Feb 2025 23:01:00 +0100 Subject: [PATCH 3/5] Delete by count per user --- Common/Constants/HardLimits.cs | 2 + Cron/Jobs/ClearOldShockerControlLogs.cs | 49 ++++++++++++++----------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/Common/Constants/HardLimits.cs b/Common/Constants/HardLimits.cs index 34c1063d..468218d3 100644 --- a/Common/Constants/HardLimits.cs +++ b/Common/Constants/HardLimits.cs @@ -47,4 +47,6 @@ public static class HardLimits public const int ShockerControlLogCustomNameMaxLength = 64; public const int CreateShareRequestMaxShockers = 128; + + public const int MaxShockerControlLogsPerUser = 2048; } diff --git a/Cron/Jobs/ClearOldShockerControlLogs.cs b/Cron/Jobs/ClearOldShockerControlLogs.cs index 7e265575..dd8499e0 100644 --- a/Cron/Jobs/ClearOldShockerControlLogs.cs +++ b/Cron/Jobs/ClearOldShockerControlLogs.cs @@ -14,7 +14,6 @@ public sealed class ClearOldShockerControlLogs { private readonly OpenShockContext _db; private readonly ILogger _logger; - private const int MaxLogCount = 1_000_000; /// /// DI constructor @@ -39,28 +38,34 @@ public async Task Execute() _logger.LogInformation("Deleted {deletedCount} shocker control logs older than {retentionThreshold:O}.", deletedByAge, retentionThreshold); - // Check the remaining number of logs to ensure it does not exceed the maximum allowed. - var remainingLogsCount = await _db.ShockerControlLogs.CountAsync(); - - if (remainingLogsCount > MaxLogCount) - { - _logger.LogInformation("Log count exceeds the maximum limit: {RemainingLogsCount} logs present, but the limit is {MaxLogCount}.", remainingLogsCount, MaxLogCount); - - // Identify the timestamp of the oldest log that should be retained to meet the log count limit. - var oldestLogToKeep = await _db.ShockerControlLogs - .OrderByDescending(log => log.CreatedOn) - .Select(log => log.CreatedOn) - .Skip(MaxLogCount) - .FirstAsync(); - - _logger.LogInformation("Preparing to delete logs created before {OldestLogToKeep:O} to enforce the log count limit.", oldestLogToKeep); + var userLogsCounts = await _db.ShockerControlLogs + .GroupBy(log => log.Shocker.DeviceNavigation.Owner) + .Select(group => new + { + UserId = group.Key, + Count = group.Count(), + DeleteBefore = _db.ShockerControlLogs + .Where(log => log.Shocker.DeviceNavigation.Owner == group.Key) + .OrderByDescending(log => log.CreatedOn) + .Skip(HardLimits.MaxShockerControlLogsPerUser) + .Take(1) + .FirstOrDefault() + }) + .Where(log => log.DeleteBefore != null) + .ToArrayAsync(); - // Delete logs that were created before the identified cutoff date. - var deletedByCount = await _db.ShockerControlLogs - .Where(log => log.CreatedOn < oldestLogToKeep) - .ExecuteDeleteAsync(); - - _logger.LogInformation("Deleted {DeletedByCount} additional logs older than {OldestLogToKeep:O}.", deletedByCount, oldestLogToKeep); + if (userLogsCounts.Length != 0) + { + _logger.LogInformation("A total of {totalLogsToDelete} logs will be deleted to enforce per-user limits.", userLogsCounts.Sum(x => x.Count)); + + foreach (var userLogCount in userLogsCounts) + { + await _db.ShockerControlLogs + .Where(log => log.Shocker.DeviceNavigation.Owner == userLogCount.UserId && log.CreatedOn < userLogCount.DeleteBefore!.CreatedOn) + .ExecuteDeleteAsync(); + } } + + _logger.LogInformation("Done!"); } } \ No newline at end of file From 6e8a4c1784b75446afaff12de28a447dfd0bafe7 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Wed, 5 Feb 2025 23:15:05 +0100 Subject: [PATCH 4/5] Account for a min value of a thousand logs --- Cron/Jobs/ClearOldShockerControlLogs.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cron/Jobs/ClearOldShockerControlLogs.cs b/Cron/Jobs/ClearOldShockerControlLogs.cs index dd8499e0..d4387770 100644 --- a/Cron/Jobs/ClearOldShockerControlLogs.cs +++ b/Cron/Jobs/ClearOldShockerControlLogs.cs @@ -43,7 +43,7 @@ public async Task Execute() .Select(group => new { UserId = group.Key, - Count = group.Count(), + CountToDelete = Math.Max(0, group.Count() - HardLimits.MaxShockerControlLogsPerUser), DeleteBefore = _db.ShockerControlLogs .Where(log => log.Shocker.DeviceNavigation.Owner == group.Key) .OrderByDescending(log => log.CreatedOn) @@ -56,7 +56,7 @@ public async Task Execute() if (userLogsCounts.Length != 0) { - _logger.LogInformation("A total of {totalLogsToDelete} logs will be deleted to enforce per-user limits.", userLogsCounts.Sum(x => x.Count)); + _logger.LogInformation("A total of {totalLogsToDelete} logs will be deleted to enforce per-user limits.", userLogsCounts.Sum(x => x.CountToDelete)); foreach (var userLogCount in userLogsCounts) { From 0766948406cfd67234844ebd9ecb80ec0f36fec2 Mon Sep 17 00:00:00 2001 From: hhvrc Date: Thu, 6 Feb 2025 00:17:53 +0100 Subject: [PATCH 5/5] Fix delete by count per user --- Cron/Jobs/ClearOldShockerControlLogs.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Cron/Jobs/ClearOldShockerControlLogs.cs b/Cron/Jobs/ClearOldShockerControlLogs.cs index d4387770..446a55ea 100644 --- a/Cron/Jobs/ClearOldShockerControlLogs.cs +++ b/Cron/Jobs/ClearOldShockerControlLogs.cs @@ -44,14 +44,13 @@ public async Task Execute() { UserId = group.Key, CountToDelete = Math.Max(0, group.Count() - HardLimits.MaxShockerControlLogsPerUser), - DeleteBefore = _db.ShockerControlLogs - .Where(log => log.Shocker.DeviceNavigation.Owner == group.Key) + DeleteBeforeDate = group .OrderByDescending(log => log.CreatedOn) .Skip(HardLimits.MaxShockerControlLogsPerUser) - .Take(1) + .Select(log => log.CreatedOn) .FirstOrDefault() }) - .Where(log => log.DeleteBefore != null) + .Where(result => result.CountToDelete > 0) .ToArrayAsync(); if (userLogsCounts.Length != 0) @@ -61,7 +60,7 @@ public async Task Execute() foreach (var userLogCount in userLogsCounts) { await _db.ShockerControlLogs - .Where(log => log.Shocker.DeviceNavigation.Owner == userLogCount.UserId && log.CreatedOn < userLogCount.DeleteBefore!.CreatedOn) + .Where(log => log.Shocker.DeviceNavigation.Owner == userLogCount.UserId && log.CreatedOn < userLogCount.DeleteBeforeDate) .ExecuteDeleteAsync(); } }