Refactor and update test projects, remove obsolete tests, and upgrade dependencies
- Deleted obsolete test files for SchedulerAuditService and SchedulerMongoSessionFactory. - Removed unused TestDataFactory class. - Updated project files for Mongo.Tests to remove references to deleted files. - Upgraded BouncyCastle.Cryptography package to version 2.6.2 across multiple projects. - Replaced Microsoft.Extensions.Http.Polly with Microsoft.Extensions.Http.Resilience in Zastava.Webhook project. - Updated NetEscapades.Configuration.Yaml package to version 3.1.0 in Configuration library. - Upgraded Pkcs11Interop package to version 5.1.2 in Cryptography libraries. - Refactored Argon2idPasswordHasher to use BouncyCastle for hashing instead of Konscious. - Updated JsonSchema.Net package to version 7.3.2 in Microservice project. - Updated global.json to use .NET SDK version 10.0.101.
This commit is contained in:
master
@@ -1,456 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Notifier.Worker.Observability;
|
||||
|
||||
/// <summary>
|
||||
/// Manages data retention policies for notifications and related data.
|
||||
/// </summary>
|
||||
public interface IRetentionPolicyService
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets all retention policies for a tenant.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<RetentionPolicy>> GetPoliciesAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets a specific retention policy.
|
||||
/// </summary>
|
||||
Task<RetentionPolicy?> GetPolicyAsync(string tenantId, string policyId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates or updates a retention policy.
|
||||
/// </summary>
|
||||
Task<RetentionPolicy> UpsertPolicyAsync(RetentionPolicy policy, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a retention policy.
|
||||
/// </summary>
|
||||
Task<bool> DeletePolicyAsync(string tenantId, string policyId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Applies retention policies and purges old data.
|
||||
/// </summary>
|
||||
Task<RetentionResult> ApplyAsync(string? tenantId = null, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets retention statistics.
|
||||
/// </summary>
|
||||
Task<RetentionStats> GetStatsAsync(string? tenantId = null, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Previews what would be deleted by retention policies.
|
||||
/// </summary>
|
||||
Task<RetentionPreview> PreviewAsync(string tenantId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A data retention policy.
|
||||
/// </summary>
|
||||
public sealed record RetentionPolicy
|
||||
{
|
||||
public required string PolicyId { get; init; }
|
||||
public required string TenantId { get; init; }
|
||||
public required string Name { get; init; }
|
||||
public string? Description { get; init; }
|
||||
public required RetentionDataType DataType { get; init; }
|
||||
public required TimeSpan RetentionPeriod { get; init; }
|
||||
public RetentionAction Action { get; init; } = RetentionAction.Delete;
|
||||
public string? ArchiveDestination { get; init; }
|
||||
public bool Enabled { get; init; } = true;
|
||||
public IReadOnlyList<string>? ChannelTypes { get; init; }
|
||||
public IReadOnlyList<string>? EventKinds { get; init; }
|
||||
public int? MinimumCount { get; init; }
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
public DateTimeOffset? LastAppliedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of data subject to retention.
|
||||
/// </summary>
|
||||
public enum RetentionDataType
|
||||
{
|
||||
Deliveries,
|
||||
DeadLetters,
|
||||
Incidents,
|
||||
AuditLogs,
|
||||
Metrics,
|
||||
Templates,
|
||||
EscalationHistory,
|
||||
DigestHistory,
|
||||
InboxNotifications
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Action to take when retention period expires.
|
||||
/// </summary>
|
||||
public enum RetentionAction
|
||||
{
|
||||
Delete,
|
||||
Archive,
|
||||
Anonymize
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of applying retention policies.
|
||||
/// </summary>
|
||||
public sealed record RetentionResult
|
||||
{
|
||||
public DateTimeOffset Timestamp { get; init; }
|
||||
public string? TenantId { get; init; }
|
||||
public int PoliciesApplied { get; init; }
|
||||
public int TotalDeleted { get; init; }
|
||||
public int TotalArchived { get; init; }
|
||||
public int TotalAnonymized { get; init; }
|
||||
public TimeSpan Duration { get; init; }
|
||||
public IReadOnlyList<RetentionPolicyResult> PolicyResults { get; init; } = [];
|
||||
public IReadOnlyList<string> Errors { get; init; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of applying a single retention policy.
|
||||
/// </summary>
|
||||
public sealed record RetentionPolicyResult
|
||||
{
|
||||
public required string PolicyId { get; init; }
|
||||
public required string PolicyName { get; init; }
|
||||
public required RetentionDataType DataType { get; init; }
|
||||
public int AffectedCount { get; init; }
|
||||
public RetentionAction ActionTaken { get; init; }
|
||||
public bool Success { get; init; }
|
||||
public string? Error { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Statistics about retention.
|
||||
/// </summary>
|
||||
public sealed record RetentionStats
|
||||
{
|
||||
public DateTimeOffset Timestamp { get; init; }
|
||||
public string? TenantId { get; init; }
|
||||
public int TotalPolicies { get; init; }
|
||||
public int EnabledPolicies { get; init; }
|
||||
public int DisabledPolicies { get; init; }
|
||||
public long TotalDeletedAllTime { get; init; }
|
||||
public long TotalArchivedAllTime { get; init; }
|
||||
public DateTimeOffset? LastRunAt { get; init; }
|
||||
public DateTimeOffset? NextScheduledRun { get; init; }
|
||||
public IReadOnlyDictionary<RetentionDataType, DataTypeStats> ByDataType { get; init; } = new Dictionary<RetentionDataType, DataTypeStats>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Statistics for a specific data type.
|
||||
/// </summary>
|
||||
public sealed record DataTypeStats
|
||||
{
|
||||
public required RetentionDataType DataType { get; init; }
|
||||
public long CurrentCount { get; init; }
|
||||
public DateTimeOffset? OldestRecord { get; init; }
|
||||
public long DeletedCount { get; init; }
|
||||
public long ArchivedCount { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Preview of what retention would delete.
|
||||
/// </summary>
|
||||
public sealed record RetentionPreview
|
||||
{
|
||||
public DateTimeOffset Timestamp { get; init; }
|
||||
public string? TenantId { get; init; }
|
||||
public int TotalToDelete { get; init; }
|
||||
public int TotalToArchive { get; init; }
|
||||
public int TotalToAnonymize { get; init; }
|
||||
public IReadOnlyList<RetentionPreviewItem> Items { get; init; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Preview item for a single policy.
|
||||
/// </summary>
|
||||
public sealed record RetentionPreviewItem
|
||||
{
|
||||
public required string PolicyId { get; init; }
|
||||
public required string PolicyName { get; init; }
|
||||
public required RetentionDataType DataType { get; init; }
|
||||
public int AffectedCount { get; init; }
|
||||
public RetentionAction Action { get; init; }
|
||||
public DateTimeOffset? OldestAffected { get; init; }
|
||||
public DateTimeOffset? NewestAffected { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for retention service.
|
||||
/// </summary>
|
||||
public sealed class RetentionOptions
|
||||
{
|
||||
public const string SectionName = "Notifier:Observability:Retention";
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
public TimeSpan DefaultRetentionPeriod { get; set; } = TimeSpan.FromDays(90);
|
||||
public TimeSpan MinimumRetentionPeriod { get; set; } = TimeSpan.FromDays(1);
|
||||
public TimeSpan MaximumRetentionPeriod { get; set; } = TimeSpan.FromDays(365 * 7);
|
||||
public bool AutoRun { get; set; } = true;
|
||||
public TimeSpan RunInterval { get; set; } = TimeSpan.FromHours(24);
|
||||
public TimeSpan RunTime { get; set; } = TimeSpan.FromHours(3);
|
||||
public int BatchSize { get; set; } = 1000;
|
||||
public bool DryRunByDefault { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of retention policy service.
|
||||
/// </summary>
|
||||
public sealed class InMemoryRetentionPolicyService : IRetentionPolicyService
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, List<RetentionPolicy>> _policies = new();
|
||||
private readonly ConcurrentDictionary<string, RetentionStats> _stats = new();
|
||||
private readonly RetentionOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<InMemoryRetentionPolicyService> _logger;
|
||||
|
||||
public InMemoryRetentionPolicyService(
|
||||
IOptions<RetentionOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<InMemoryRetentionPolicyService> logger)
|
||||
{
|
||||
_options = options?.Value ?? new RetentionOptions();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<RetentionPolicy>> GetPoliciesAsync(string tenantId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_policies.TryGetValue(tenantId, out var policies))
|
||||
return Task.FromResult<IReadOnlyList<RetentionPolicy>>([]);
|
||||
return Task.FromResult<IReadOnlyList<RetentionPolicy>>(policies.ToList());
|
||||
}
|
||||
|
||||
public Task<RetentionPolicy?> GetPolicyAsync(string tenantId, string policyId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_policies.TryGetValue(tenantId, out var policies))
|
||||
return Task.FromResult<RetentionPolicy?>(null);
|
||||
return Task.FromResult(policies.FirstOrDefault(p => p.PolicyId == policyId));
|
||||
}
|
||||
|
||||
public Task<RetentionPolicy> UpsertPolicyAsync(RetentionPolicy policy, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var list = _policies.GetOrAdd(policy.TenantId, _ => []);
|
||||
|
||||
lock (list)
|
||||
{
|
||||
var index = list.FindIndex(p => p.PolicyId == policy.PolicyId);
|
||||
var updated = policy with { UpdatedAt = now, CreatedAt = index < 0 ? now : list[index].CreatedAt };
|
||||
if (index >= 0) list[index] = updated;
|
||||
else list.Add(updated);
|
||||
_logger.LogInformation("Upserted retention policy {PolicyId} for tenant {TenantId}", policy.PolicyId, policy.TenantId);
|
||||
return Task.FromResult(updated);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<bool> DeletePolicyAsync(string tenantId, string policyId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_policies.TryGetValue(tenantId, out var policies)) return Task.FromResult(false);
|
||||
lock (policies)
|
||||
{
|
||||
var removed = policies.RemoveAll(p => p.PolicyId == policyId) > 0;
|
||||
if (removed) _logger.LogInformation("Deleted retention policy {PolicyId} for tenant {TenantId}", policyId, tenantId);
|
||||
return Task.FromResult(removed);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<RetentionResult> ApplyAsync(string? tenantId = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var startTime = _timeProvider.GetUtcNow();
|
||||
var policyResults = new List<RetentionPolicyResult>();
|
||||
var errors = new List<string>();
|
||||
var totalDeleted = 0;
|
||||
var totalArchived = 0;
|
||||
var totalAnonymized = 0;
|
||||
|
||||
var tenantsToProcess = tenantId is not null ? [tenantId] : _policies.Keys.ToList();
|
||||
|
||||
foreach (var t in tenantsToProcess)
|
||||
{
|
||||
if (!_policies.TryGetValue(t, out var policies)) continue;
|
||||
|
||||
foreach (var policy in policies.Where(p => p.Enabled))
|
||||
{
|
||||
try
|
||||
{
|
||||
var affectedCount = SimulateRetention(policy);
|
||||
var result = new RetentionPolicyResult
|
||||
{
|
||||
PolicyId = policy.PolicyId,
|
||||
PolicyName = policy.Name,
|
||||
DataType = policy.DataType,
|
||||
AffectedCount = affectedCount,
|
||||
ActionTaken = policy.Action,
|
||||
Success = true
|
||||
};
|
||||
policyResults.Add(result);
|
||||
|
||||
switch (policy.Action)
|
||||
{
|
||||
case RetentionAction.Delete: totalDeleted += affectedCount; break;
|
||||
case RetentionAction.Archive: totalArchived += affectedCount; break;
|
||||
case RetentionAction.Anonymize: totalAnonymized += affectedCount; break;
|
||||
}
|
||||
|
||||
// Update last applied time
|
||||
lock (policies)
|
||||
{
|
||||
var idx = policies.FindIndex(p => p.PolicyId == policy.PolicyId);
|
||||
if (idx >= 0) policies[idx] = policy with { LastAppliedAt = _timeProvider.GetUtcNow() };
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
errors.Add($"Policy {policy.PolicyId}: {ex.Message}");
|
||||
policyResults.Add(new RetentionPolicyResult
|
||||
{
|
||||
PolicyId = policy.PolicyId,
|
||||
PolicyName = policy.Name,
|
||||
DataType = policy.DataType,
|
||||
Success = false,
|
||||
Error = ex.Message
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var endTime = _timeProvider.GetUtcNow();
|
||||
_logger.LogInformation("Applied retention policies: {Deleted} deleted, {Archived} archived, {Anonymized} anonymized", totalDeleted, totalArchived, totalAnonymized);
|
||||
|
||||
return Task.FromResult(new RetentionResult
|
||||
{
|
||||
Timestamp = endTime,
|
||||
TenantId = tenantId,
|
||||
PoliciesApplied = policyResults.Count(r => r.Success),
|
||||
TotalDeleted = totalDeleted,
|
||||
TotalArchived = totalArchived,
|
||||
TotalAnonymized = totalAnonymized,
|
||||
Duration = endTime - startTime,
|
||||
PolicyResults = policyResults,
|
||||
Errors = errors
|
||||
});
|
||||
}
|
||||
|
||||
public Task<RetentionStats> GetStatsAsync(string? tenantId = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var allPolicies = tenantId is not null
|
||||
? (_policies.TryGetValue(tenantId, out var p) ? p : [])
|
||||
: _policies.Values.SelectMany(v => v).ToList();
|
||||
|
||||
var byDataType = Enum.GetValues<RetentionDataType>()
|
||||
.ToDictionary(dt => dt, dt => new DataTypeStats { DataType = dt, CurrentCount = 0, DeletedCount = 0, ArchivedCount = 0 });
|
||||
|
||||
return Task.FromResult(new RetentionStats
|
||||
{
|
||||
Timestamp = _timeProvider.GetUtcNow(),
|
||||
TenantId = tenantId,
|
||||
TotalPolicies = allPolicies.Count,
|
||||
EnabledPolicies = allPolicies.Count(p => p.Enabled),
|
||||
DisabledPolicies = allPolicies.Count(p => !p.Enabled),
|
||||
LastRunAt = allPolicies.Max(p => p.LastAppliedAt),
|
||||
ByDataType = byDataType
|
||||
});
|
||||
}
|
||||
|
||||
public Task<RetentionPreview> PreviewAsync(string tenantId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_policies.TryGetValue(tenantId, out var policies))
|
||||
return Task.FromResult(new RetentionPreview { Timestamp = _timeProvider.GetUtcNow(), TenantId = tenantId });
|
||||
|
||||
var items = policies.Where(p => p.Enabled).Select(p => new RetentionPreviewItem
|
||||
{
|
||||
PolicyId = p.PolicyId,
|
||||
PolicyName = p.Name,
|
||||
DataType = p.DataType,
|
||||
AffectedCount = SimulateRetention(p),
|
||||
Action = p.Action
|
||||
}).ToList();
|
||||
|
||||
return Task.FromResult(new RetentionPreview
|
||||
{
|
||||
Timestamp = _timeProvider.GetUtcNow(),
|
||||
TenantId = tenantId,
|
||||
TotalToDelete = items.Where(i => i.Action == RetentionAction.Delete).Sum(i => i.AffectedCount),
|
||||
TotalToArchive = items.Where(i => i.Action == RetentionAction.Archive).Sum(i => i.AffectedCount),
|
||||
TotalToAnonymize = items.Where(i => i.Action == RetentionAction.Anonymize).Sum(i => i.AffectedCount),
|
||||
Items = items
|
||||
});
|
||||
}
|
||||
|
||||
private int SimulateRetention(RetentionPolicy policy)
|
||||
{
|
||||
// In production, this would query actual data stores
|
||||
// For simulation, return a random count based on retention period
|
||||
var daysFactor = (int)policy.RetentionPeriod.TotalDays;
|
||||
return Math.Max(0, 100 - daysFactor);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Background service that runs retention policies on schedule.
|
||||
/// </summary>
|
||||
public sealed class RetentionPolicyRunner : BackgroundService
|
||||
{
|
||||
private readonly IRetentionPolicyService _retentionService;
|
||||
private readonly RetentionOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<RetentionPolicyRunner> _logger;
|
||||
|
||||
public RetentionPolicyRunner(
|
||||
IRetentionPolicyService retentionService,
|
||||
IOptions<RetentionOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<RetentionPolicyRunner> logger)
|
||||
{
|
||||
_retentionService = retentionService ?? throw new ArgumentNullException(nameof(retentionService));
|
||||
_options = options?.Value ?? new RetentionOptions();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!_options.Enabled || !_options.AutoRun)
|
||||
{
|
||||
_logger.LogInformation("Retention policy runner is disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Retention policy runner started with interval {Interval}", _options.RunInterval);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var nextRun = now.Date.Add(_options.RunTime);
|
||||
if (nextRun <= now) nextRun = nextRun.AddDays(1);
|
||||
|
||||
var delay = nextRun - now;
|
||||
if (delay > _options.RunInterval) delay = _options.RunInterval;
|
||||
|
||||
await Task.Delay(delay, stoppingToken);
|
||||
|
||||
_logger.LogInformation("Running scheduled retention policy application");
|
||||
var result = await _retentionService.ApplyAsync(cancellationToken: stoppingToken);
|
||||
_logger.LogInformation("Retention completed: {Deleted} deleted, {Archived} archived in {Duration}ms",
|
||||
result.TotalDeleted, result.TotalArchived, result.Duration.TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error running retention policies");
|
||||
await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user