audit work, fixed StellaOps.sln warnings/errors, fixed tests, sprints work, new advisories

This commit is contained in:
master
2026-01-07 18:49:59 +02:00
parent 04ec098046
commit 608a7f85c0
866 changed files with 56323 additions and 6231 deletions

View File

@@ -0,0 +1,86 @@
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
using StellaOps.Eventing.Models;
using StellaOps.HybridLogicalClock;
namespace StellaOps.Eventing.Storage;
/// <summary>
/// Interface for timeline event persistence.
/// </summary>
public interface ITimelineEventStore
{
/// <summary>
/// Appends a single event to the store.
/// </summary>
/// <param name="timelineEvent">The event to append.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A task representing the operation.</returns>
Task AppendAsync(TimelineEvent timelineEvent, CancellationToken cancellationToken = default);
/// <summary>
/// Appends multiple events atomically.
/// </summary>
/// <param name="events">The events to append.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A task representing the operation.</returns>
Task AppendBatchAsync(IEnumerable<TimelineEvent> events, CancellationToken cancellationToken = default);
/// <summary>
/// Gets events by correlation ID, ordered by HLC timestamp.
/// </summary>
/// <param name="correlationId">The correlation ID.</param>
/// <param name="limit">Maximum number of events to return.</param>
/// <param name="offset">Number of events to skip.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Events ordered by HLC timestamp.</returns>
Task<IReadOnlyList<TimelineEvent>> GetByCorrelationIdAsync(
string correlationId,
int limit = 100,
int offset = 0,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets events within an HLC range.
/// </summary>
/// <param name="correlationId">The correlation ID.</param>
/// <param name="fromHlc">Start of HLC range (inclusive).</param>
/// <param name="toHlc">End of HLC range (inclusive).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Events within the range, ordered by HLC timestamp.</returns>
Task<IReadOnlyList<TimelineEvent>> GetByHlcRangeAsync(
string correlationId,
HlcTimestamp fromHlc,
HlcTimestamp toHlc,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets events by service.
/// </summary>
/// <param name="service">The service name.</param>
/// <param name="fromHlc">Optional start of HLC range.</param>
/// <param name="limit">Maximum number of events to return.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Events from the service, ordered by HLC timestamp.</returns>
Task<IReadOnlyList<TimelineEvent>> GetByServiceAsync(
string service,
HlcTimestamp? fromHlc = null,
int limit = 100,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets a single event by ID.
/// </summary>
/// <param name="eventId">The event ID.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The event, or null if not found.</returns>
Task<TimelineEvent?> GetByIdAsync(string eventId, CancellationToken cancellationToken = default);
/// <summary>
/// Counts events for a correlation ID.
/// </summary>
/// <param name="correlationId">The correlation ID.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Number of events.</returns>
Task<long> CountByCorrelationIdAsync(string correlationId, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,126 @@
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
using System.Collections.Concurrent;
using StellaOps.Eventing.Models;
using StellaOps.HybridLogicalClock;
namespace StellaOps.Eventing.Storage;
/// <summary>
/// In-memory implementation of <see cref="ITimelineEventStore"/> for testing.
/// </summary>
public sealed class InMemoryTimelineEventStore : ITimelineEventStore
{
private readonly ConcurrentDictionary<string, TimelineEvent> _events = new();
/// <inheritdoc/>
public Task AppendAsync(TimelineEvent timelineEvent, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(timelineEvent);
_events.TryAdd(timelineEvent.EventId, timelineEvent);
return Task.CompletedTask;
}
/// <inheritdoc/>
public Task AppendBatchAsync(IEnumerable<TimelineEvent> events, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(events);
foreach (var e in events)
{
_events.TryAdd(e.EventId, e);
}
return Task.CompletedTask;
}
/// <inheritdoc/>
public Task<IReadOnlyList<TimelineEvent>> GetByCorrelationIdAsync(
string correlationId,
int limit = 100,
int offset = 0,
CancellationToken cancellationToken = default)
{
var result = _events.Values
.Where(e => e.CorrelationId == correlationId)
.OrderBy(e => e.THlc.ToSortableString())
.Skip(offset)
.Take(limit)
.ToList();
return Task.FromResult<IReadOnlyList<TimelineEvent>>(result);
}
/// <inheritdoc/>
public Task<IReadOnlyList<TimelineEvent>> GetByHlcRangeAsync(
string correlationId,
HlcTimestamp fromHlc,
HlcTimestamp toHlc,
CancellationToken cancellationToken = default)
{
var fromStr = fromHlc.ToSortableString();
var toStr = toHlc.ToSortableString();
var result = _events.Values
.Where(e => e.CorrelationId == correlationId)
.Where(e =>
{
var hlcStr = e.THlc.ToSortableString();
return string.Compare(hlcStr, fromStr, StringComparison.Ordinal) >= 0 &&
string.Compare(hlcStr, toStr, StringComparison.Ordinal) <= 0;
})
.OrderBy(e => e.THlc.ToSortableString())
.ToList();
return Task.FromResult<IReadOnlyList<TimelineEvent>>(result);
}
/// <inheritdoc/>
public Task<IReadOnlyList<TimelineEvent>> GetByServiceAsync(
string service,
HlcTimestamp? fromHlc = null,
int limit = 100,
CancellationToken cancellationToken = default)
{
var query = _events.Values.Where(e => e.Service == service);
if (fromHlc.HasValue)
{
var fromStr = fromHlc.Value.ToSortableString();
query = query.Where(e =>
string.Compare(e.THlc.ToSortableString(), fromStr, StringComparison.Ordinal) >= 0);
}
var result = query
.OrderBy(e => e.THlc.ToSortableString())
.Take(limit)
.ToList();
return Task.FromResult<IReadOnlyList<TimelineEvent>>(result);
}
/// <inheritdoc/>
public Task<TimelineEvent?> GetByIdAsync(string eventId, CancellationToken cancellationToken = default)
{
_events.TryGetValue(eventId, out var e);
return Task.FromResult(e);
}
/// <inheritdoc/>
public Task<long> CountByCorrelationIdAsync(string correlationId, CancellationToken cancellationToken = default)
{
var count = _events.Values.Count(e => e.CorrelationId == correlationId);
return Task.FromResult((long)count);
}
/// <summary>
/// Clears all events (for testing).
/// </summary>
public void Clear() => _events.Clear();
/// <summary>
/// Gets all events (for testing).
/// </summary>
public IReadOnlyCollection<TimelineEvent> GetAll() => _events.Values.ToList();
}

View File

@@ -0,0 +1,312 @@
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
using System.Data;
using System.Globalization;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Eventing.Models;
using StellaOps.HybridLogicalClock;
namespace StellaOps.Eventing.Storage;
/// <summary>
/// PostgreSQL implementation of <see cref="ITimelineEventStore"/>.
/// </summary>
public sealed class PostgresTimelineEventStore : ITimelineEventStore
{
private readonly NpgsqlDataSource _dataSource;
private readonly ILogger<PostgresTimelineEventStore> _logger;
/// <summary>
/// Initializes a new instance of the <see cref="PostgresTimelineEventStore"/> class.
/// </summary>
public PostgresTimelineEventStore(
NpgsqlDataSource dataSource,
ILogger<PostgresTimelineEventStore> logger)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <inheritdoc/>
public async Task AppendAsync(TimelineEvent timelineEvent, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(timelineEvent);
const string sql = """
INSERT INTO timeline.events (
event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
) VALUES (
@event_id, @t_hlc, @ts_wall, @service, @trace_parent,
@correlation_id, @kind, @payload::jsonb, @payload_digest,
@engine_name, @engine_version, @engine_digest, @dsse_sig, @schema_version
)
ON CONFLICT (event_id) DO NOTHING
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
AddEventParameters(command, timelineEvent);
var rowsAffected = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
if (rowsAffected == 0)
{
_logger.LogDebug("Event {EventId} already exists (idempotent insert)", timelineEvent.EventId);
}
}
/// <inheritdoc/>
public async Task AppendBatchAsync(IEnumerable<TimelineEvent> events, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(events);
var eventList = events.ToList();
if (eventList.Count == 0)
{
return;
}
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(IsolationLevel.ReadCommitted, cancellationToken).ConfigureAwait(false);
try
{
const string sql = """
INSERT INTO timeline.events (
event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
) VALUES (
@event_id, @t_hlc, @ts_wall, @service, @trace_parent,
@correlation_id, @kind, @payload::jsonb, @payload_digest,
@engine_name, @engine_version, @engine_digest, @dsse_sig, @schema_version
)
ON CONFLICT (event_id) DO NOTHING
""";
foreach (var timelineEvent in eventList)
{
await using var command = new NpgsqlCommand(sql, connection, transaction);
AddEventParameters(command, timelineEvent);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Appended batch of {Count} events", eventList.Count);
}
catch
{
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
throw;
}
}
/// <inheritdoc/>
public async Task<IReadOnlyList<TimelineEvent>> GetByCorrelationIdAsync(
string correlationId,
int limit = 100,
int offset = 0,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(correlationId);
const string sql = """
SELECT event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
FROM timeline.events
WHERE correlation_id = @correlation_id
ORDER BY t_hlc ASC
LIMIT @limit OFFSET @offset
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("@correlation_id", correlationId);
command.Parameters.AddWithValue("@limit", limit);
command.Parameters.AddWithValue("@offset", offset);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc/>
public async Task<IReadOnlyList<TimelineEvent>> GetByHlcRangeAsync(
string correlationId,
HlcTimestamp fromHlc,
HlcTimestamp toHlc,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(correlationId);
const string sql = """
SELECT event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
FROM timeline.events
WHERE correlation_id = @correlation_id
AND t_hlc >= @from_hlc
AND t_hlc <= @to_hlc
ORDER BY t_hlc ASC
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("@correlation_id", correlationId);
command.Parameters.AddWithValue("@from_hlc", fromHlc.ToSortableString());
command.Parameters.AddWithValue("@to_hlc", toHlc.ToSortableString());
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc/>
public async Task<IReadOnlyList<TimelineEvent>> GetByServiceAsync(
string service,
HlcTimestamp? fromHlc = null,
int limit = 100,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(service);
var sql = fromHlc.HasValue
? """
SELECT event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
FROM timeline.events
WHERE service = @service AND t_hlc >= @from_hlc
ORDER BY t_hlc ASC
LIMIT @limit
"""
: """
SELECT event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
FROM timeline.events
WHERE service = @service
ORDER BY t_hlc ASC
LIMIT @limit
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("@service", service);
command.Parameters.AddWithValue("@limit", limit);
if (fromHlc.HasValue)
{
command.Parameters.AddWithValue("@from_hlc", fromHlc.Value.ToSortableString());
}
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc/>
public async Task<TimelineEvent?> GetByIdAsync(string eventId, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(eventId);
const string sql = """
SELECT event_id, t_hlc, ts_wall, service, trace_parent,
correlation_id, kind, payload, payload_digest,
engine_name, engine_version, engine_digest, dsse_sig, schema_version
FROM timeline.events
WHERE event_id = @event_id
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("@event_id", eventId);
var events = await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
return events.Count > 0 ? events[0] : null;
}
/// <inheritdoc/>
public async Task<long> CountByCorrelationIdAsync(string correlationId, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(correlationId);
const string sql = """
SELECT COUNT(*) FROM timeline.events WHERE correlation_id = @correlation_id
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("@correlation_id", correlationId);
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt64(result, CultureInfo.InvariantCulture);
}
private static void AddEventParameters(NpgsqlCommand command, TimelineEvent e)
{
command.Parameters.AddWithValue("@event_id", e.EventId);
command.Parameters.AddWithValue("@t_hlc", e.THlc.ToSortableString());
command.Parameters.AddWithValue("@ts_wall", e.TsWall);
command.Parameters.AddWithValue("@service", e.Service);
command.Parameters.AddWithValue("@trace_parent", (object?)e.TraceParent ?? DBNull.Value);
command.Parameters.AddWithValue("@correlation_id", e.CorrelationId);
command.Parameters.AddWithValue("@kind", e.Kind);
command.Parameters.AddWithValue("@payload", e.Payload);
command.Parameters.AddWithValue("@payload_digest", e.PayloadDigest);
command.Parameters.AddWithValue("@engine_name", e.EngineVersion.EngineName);
command.Parameters.AddWithValue("@engine_version", e.EngineVersion.Version);
command.Parameters.AddWithValue("@engine_digest", e.EngineVersion.SourceDigest);
command.Parameters.AddWithValue("@dsse_sig", (object?)e.DsseSig ?? DBNull.Value);
command.Parameters.AddWithValue("@schema_version", e.SchemaVersion);
}
private static async Task<IReadOnlyList<TimelineEvent>> ExecuteQueryAsync(
NpgsqlCommand command,
CancellationToken cancellationToken)
{
var events = new List<TimelineEvent>();
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
events.Add(MapFromReader(reader));
}
return events;
}
private static TimelineEvent MapFromReader(NpgsqlDataReader reader)
{
var hlcString = reader.GetString(reader.GetOrdinal("t_hlc"));
return new TimelineEvent
{
EventId = reader.GetString(reader.GetOrdinal("event_id")),
THlc = HlcTimestamp.Parse(hlcString),
TsWall = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("ts_wall")),
Service = reader.GetString(reader.GetOrdinal("service")),
TraceParent = reader.IsDBNull(reader.GetOrdinal("trace_parent"))
? null
: reader.GetString(reader.GetOrdinal("trace_parent")),
CorrelationId = reader.GetString(reader.GetOrdinal("correlation_id")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
Payload = reader.GetString(reader.GetOrdinal("payload")),
PayloadDigest = (byte[])reader.GetValue(reader.GetOrdinal("payload_digest")),
EngineVersion = new EngineVersionRef(
reader.GetString(reader.GetOrdinal("engine_name")),
reader.GetString(reader.GetOrdinal("engine_version")),
reader.GetString(reader.GetOrdinal("engine_digest"))),
DsseSig = reader.IsDBNull(reader.GetOrdinal("dsse_sig"))
? null
: reader.GetString(reader.GetOrdinal("dsse_sig")),
SchemaVersion = reader.GetInt32(reader.GetOrdinal("schema_version"))
};
}
}