stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

feat: Implement Filesystem and MongoDB provenance writers for PackRun execution context
Some checks failed
Airgap Sealed CI Smoke / sealed-smoke (push) Has been cancelled
Details
Docs CI / lint-and-preview (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details

Browse Source 
- Added `FilesystemPackRunProvenanceWriter` to write provenance manifests to the filesystem.
- Introduced `MongoPackRunArtifactReader` to read artifacts from MongoDB.
- Created `MongoPackRunProvenanceWriter` to store provenance manifests in MongoDB.
- Developed unit tests for filesystem and MongoDB provenance writers.
- Established `ITimelineEventStore` and `ITimelineIngestionService` interfaces for timeline event handling.
- Implemented `TimelineIngestionService` to validate and persist timeline events with hashing.
- Created PostgreSQL schema and migration scripts for timeline indexing.
- Added dependency injection support for timeline indexer services.
- Developed tests for timeline ingestion and schema validation.
This commit is contained in:
StellaOps Bot
2025-11-30 15:38:14 +02:00
parent 8f54ffa203
commit 17d45a6d30
276 changed files with 8618 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Abstractions/ITimelineEventStore.cs Normal file
View File

@@ -0,0 +1,16 @@
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Core.Abstractions;
/// <summary>
/// Persistence contract for timeline event ingestion.
/// Implementations must enforce tenant isolation and idempotency on (tenant_id, event_id).
/// </summary>
public interface ITimelineEventStore
{
/// <summary>
/// Inserts the event atomically (headers, payloads, digests).
/// Must be idempotent on (tenant_id, event_id) and return whether a new row was created.
/// </summary>
Task<bool> InsertAsync(TimelineEventEnvelope envelope, CancellationToken cancellationToken = default);
}

12
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Abstractions/ITimelineEventSubscriber.cs Normal file
View File

@@ -0,0 +1,12 @@
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Core.Abstractions;
/// <summary>
/// Abstraction over transport-specific event subscriptions (NATS/Redis/etc.).
/// Implementations yield tenant-scoped timeline event envelopes in publish order.
/// </summary>
public interface ITimelineEventSubscriber
{
IAsyncEnumerable<TimelineEventEnvelope> SubscribeAsync(CancellationToken cancellationToken = default);
}

12
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Abstractions/ITimelineIngestionService.cs Normal file
View File

@@ -0,0 +1,12 @@
using StellaOps.TimelineIndexer.Core.Models;
using StellaOps.TimelineIndexer.Core.Models.Results;
namespace StellaOps.TimelineIndexer.Core.Abstractions;
/// <summary>
/// High-level ingestion service that validates, hashes, and persists timeline events.
/// </summary>
public interface ITimelineIngestionService
{
Task<TimelineIngestResult> IngestAsync(TimelineEventEnvelope envelope, CancellationToken cancellationToken = default);
}

9
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Abstractions/ITimelineQueryService.cs Normal file
View File

@@ -0,0 +1,9 @@
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Core.Abstractions;
public interface ITimelineQueryService
{
Task<IReadOnlyList<TimelineEventView>> QueryAsync(string tenantId, TimelineQueryOptions options, CancellationToken cancellationToken = default);
Task<TimelineEventView?> GetAsync(string tenantId, string eventId, CancellationToken cancellationToken = default);
}

9
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Abstractions/ITimelineQueryStore.cs Normal file
View File

@@ -0,0 +1,9 @@
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Core.Abstractions;
public interface ITimelineQueryStore
{
Task<IReadOnlyList<TimelineEventView>> QueryAsync(string tenantId, TimelineQueryOptions options, CancellationToken cancellationToken);
Task<TimelineEventView?> GetAsync(string tenantId, string eventId, CancellationToken cancellationToken);
}

6
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Class1.cs
View File

@@ -1,6 +0,0 @@
namespace StellaOps.TimelineIndexer.Core;
public class Class1
{
}

3
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Models/Results/TimelineIngestResult.cs Normal file
View File

@@ -0,0 +1,3 @@
namespace StellaOps.TimelineIndexer.Core.Models.Results;
public sealed record TimelineIngestResult(bool Inserted);

29
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Models/TimelineEventEnvelope.cs Normal file
View File

@@ -0,0 +1,29 @@
namespace StellaOps.TimelineIndexer.Core.Models;
/// <summary>
/// Canonical ingestion envelope for timeline events.
/// Maps closely to orchestrator/notify envelopes while remaining transport-agnostic.
/// </summary>
public sealed class TimelineEventEnvelope
{
public required string EventId { get; init; }
public required string TenantId { get; init; }
public required string EventType { get; init; }
public required string Source { get; init; }
public required DateTimeOffset OccurredAt { get; init; }
public string? CorrelationId { get; init; }
public string? TraceId { get; init; }
public string? Actor { get; init; }
public string Severity { get; init; } = "info";
public string? PayloadHash { get; set; }
public string RawPayloadJson { get; init; } = "{}";
public string? NormalizedPayloadJson { get; init; }
public IDictionary<string, string>? Attributes { get; init; }
public string? BundleDigest { get; init; }
public Guid? BundleId { get; init; }
public string? AttestationSubject { get; init; }
public string? AttestationDigest { get; init; }
public string? ManifestUri { get; init; }
}

20
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Models/TimelineEventView.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace StellaOps.TimelineIndexer.Core.Models;
/// <summary>
/// Projected timeline event for query responses.
/// </summary>
public sealed class TimelineEventView
{
public required long EventSeq { get; init; }
public required string EventId { get; init; }
public required string TenantId { get; init; }
public required string EventType { get; init; }
public required string Source { get; init; }
public required DateTimeOffset OccurredAt { get; init; }
public required DateTimeOffset ReceivedAt { get; init; }
public string? CorrelationId { get; init; }
public string? TraceId { get; init; }
public string? Actor { get; init; }
public string Severity { get; init; } = "info";
public string? PayloadHash { get; init; }
}

15
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Models/TimelineQueryOptions.cs Normal file
View File

@@ -0,0 +1,15 @@
namespace StellaOps.TimelineIndexer.Core.Models;
/// <summary>
/// Query filters for timeline listing.
/// </summary>
public sealed class TimelineQueryOptions
{
public string? EventType { get; init; }
public string? CorrelationId { get; init; }
public string? TraceId { get; init; }
public string? Severity { get; init; }
public DateTimeOffset? Since { get; init; }
public long? AfterEventSeq { get; init; }
public int Limit { get; init; } = 100;
}

46
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Services/TimelineIngestionService.cs Normal file
View File

@@ -0,0 +1,46 @@
using System.Security.Cryptography;
using System.Text;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
using StellaOps.TimelineIndexer.Core.Models.Results;
namespace StellaOps.TimelineIndexer.Core.Services;
/// <summary>
/// Validates and persists timeline events with deterministic hashing.
/// </summary>
public sealed class TimelineIngestionService(ITimelineEventStore store) : ITimelineIngestionService
{
public async Task<TimelineIngestResult> IngestAsync(TimelineEventEnvelope envelope, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(envelope);
Validate(envelope);
if (string.IsNullOrWhiteSpace(envelope.PayloadHash))
{
envelope.PayloadHash = ComputePayloadHash(envelope.RawPayloadJson);
}
var inserted = await store.InsertAsync(envelope, cancellationToken).ConfigureAwait(false);
return new TimelineIngestResult(inserted);
}
private static void Validate(TimelineEventEnvelope envelope)
{
if (string.IsNullOrWhiteSpace(envelope.EventId))
throw new ArgumentException("event_id is required", nameof(envelope));
if (string.IsNullOrWhiteSpace(envelope.TenantId))
throw new ArgumentException("tenant_id is required", nameof(envelope));
if (string.IsNullOrWhiteSpace(envelope.EventType))
throw new ArgumentException("event_type is required", nameof(envelope));
if (string.IsNullOrWhiteSpace(envelope.Source))
throw new ArgumentException("source is required", nameof(envelope));
}
internal static string ComputePayloadHash(string payloadJson)
{
var bytes = Encoding.UTF8.GetBytes(payloadJson ?? string.Empty);
var hash = SHA256.HashData(bytes);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
}

29
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Core/Services/TimelineQueryService.cs Normal file
View File

@@ -0,0 +1,29 @@
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Core.Services;
public sealed class TimelineQueryService(ITimelineQueryStore store) : ITimelineQueryService
{
public Task<IReadOnlyList<TimelineEventView>> QueryAsync(string tenantId, TimelineQueryOptions options, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentNullException.ThrowIfNull(options);
return store.QueryAsync(tenantId, Normalize(options), cancellationToken);
}
public Task<TimelineEventView?> GetAsync(string tenantId, string eventId, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentException.ThrowIfNullOrWhiteSpace(eventId);
return store.GetAsync(tenantId, eventId, cancellationToken);
}
private static TimelineQueryOptions Normalize(TimelineQueryOptions options)
{
var limit = options.Limit;
if (limit <= 0) limit = 100;
if (limit > 500) limit = 500;
return options with { Limit = limit };
}
}

6
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Class1.cs
View File

@@ -1,6 +0,0 @@
namespace StellaOps.TimelineIndexer.Infrastructure;
public class Class1
{
}

111
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Db/Migrations/001_initial_schema.sql Normal file
View File

@@ -0,0 +1,111 @@
-- 001_initial_schema.sql
-- Establishes Timeline Indexer schema, RLS scaffolding, and evidence linkage tables.
CREATE EXTENSION IF NOT EXISTS pgcrypto;
CREATE SCHEMA IF NOT EXISTS timeline;
CREATE SCHEMA IF NOT EXISTS timeline_app;
-- Enforce tenant context for all RLS policies
CREATE OR REPLACE FUNCTION timeline_app.require_current_tenant()
RETURNS text
LANGUAGE plpgsql
AS $$
DECLARE
tenant_text text;
BEGIN
tenant_text := current_setting('app.current_tenant', true);
IF tenant_text IS NULL OR length(tenant_text) = 0 THEN
RAISE EXCEPTION 'app.current_tenant is not set for the current session';
END IF;
RETURN tenant_text;
END;
$$;
-- Severity enum keeps ordering deterministic and compact
DO $$
BEGIN
CREATE TYPE timeline.event_severity AS ENUM ('info', 'notice', 'warn', 'error', 'critical');
EXCEPTION
WHEN duplicate_object THEN NULL;
END
$$;
-- Core event header table (dedupe + ordering)
CREATE TABLE IF NOT EXISTS timeline.timeline_events
(
event_seq bigserial PRIMARY KEY,
event_id text NOT NULL,
tenant_id text NOT NULL,
source text NOT NULL,
event_type text NOT NULL,
occurred_at timestamptz NOT NULL,
received_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
correlation_id text,
trace_id text,
actor text,
severity timeline.event_severity NOT NULL DEFAULT 'info',
payload_hash text CHECK (payload_hash IS NULL OR payload_hash ~ '^[0-9a-f]{64}$'),
attributes jsonb NOT NULL DEFAULT '{}'::jsonb,
UNIQUE (tenant_id, event_id)
);
CREATE INDEX IF NOT EXISTS ix_timeline_events_tenant_occurred
ON timeline.timeline_events (tenant_id, occurred_at DESC, event_seq DESC);
CREATE INDEX IF NOT EXISTS ix_timeline_events_type
ON timeline.timeline_events (tenant_id, event_type, occurred_at DESC);
ALTER TABLE timeline.timeline_events ENABLE ROW LEVEL SECURITY;
CREATE POLICY IF NOT EXISTS timeline_events_isolation
ON timeline.timeline_events
USING (tenant_id = timeline_app.require_current_tenant())
WITH CHECK (tenant_id = timeline_app.require_current_tenant());
-- Raw and normalized payloads per event
CREATE TABLE IF NOT EXISTS timeline.timeline_event_details
(
event_id text NOT NULL,
tenant_id text NOT NULL,
envelope_version text NOT NULL,
raw_payload jsonb NOT NULL,
normalized_payload jsonb,
created_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
CONSTRAINT fk_event_details FOREIGN KEY (event_id, tenant_id)
REFERENCES timeline.timeline_events (event_id, tenant_id) ON DELETE CASCADE,
PRIMARY KEY (event_id, tenant_id)
);
ALTER TABLE timeline.timeline_event_details ENABLE ROW LEVEL SECURITY;
CREATE POLICY IF NOT EXISTS timeline_event_details_isolation
ON timeline.timeline_event_details
USING (tenant_id = timeline_app.require_current_tenant())
WITH CHECK (tenant_id = timeline_app.require_current_tenant());
-- Evidence linkage (bundle/attestation manifests)
CREATE TABLE IF NOT EXISTS timeline.timeline_event_digests
(
digest_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id text NOT NULL,
event_id text NOT NULL,
bundle_id uuid,
bundle_digest text,
attestation_subject text,
attestation_digest text,
manifest_uri text,
created_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
CONSTRAINT fk_event_digest_event FOREIGN KEY (event_id, tenant_id)
REFERENCES timeline.timeline_events (event_id, tenant_id) ON DELETE CASCADE,
CONSTRAINT ck_bundle_digest_sha CHECK (bundle_digest IS NULL OR bundle_digest ~ '^sha256:[0-9a-f]{64}$'),
CONSTRAINT ck_attestation_digest_sha CHECK (attestation_digest IS NULL OR attestation_digest ~ '^sha256:[0-9a-f]{64}$')
);
CREATE INDEX IF NOT EXISTS ix_timeline_digests_event
ON timeline.timeline_event_digests (tenant_id, event_id);
CREATE INDEX IF NOT EXISTS ix_timeline_digests_bundle
ON timeline.timeline_event_digests (tenant_id, bundle_digest);
ALTER TABLE timeline.timeline_event_digests ENABLE ROW LEVEL SECURITY;
CREATE POLICY IF NOT EXISTS timeline_event_digests_isolation
ON timeline.timeline_event_digests
USING (tenant_id = timeline_app.require_current_tenant())
WITH CHECK (tenant_id = timeline_app.require_current_tenant());

112
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Db/TimelineEventStore.cs Normal file
View File

@@ -0,0 +1,112 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using System.Text.Json;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Infrastructure.Db;
/// <summary>
/// Postgres-backed implementation of ITimelineEventStore.
/// </summary>
public sealed class TimelineEventStore(TimelineIndexerDataSource dataSource, ILogger<TimelineEventStore> logger)
: RepositoryBase<TimelineIndexerDataSource>(dataSource, logger), ITimelineEventStore
{
private const string InsertEventSql = """
INSERT INTO timeline.timeline_events
(event_id, tenant_id, source, event_type, occurred_at, correlation_id, trace_id, actor, severity, payload_hash, attributes)
VALUES
(@event_id, @tenant_id, @source, @event_type, @occurred_at, @correlation_id, @trace_id, @actor, @severity, @payload_hash, @attributes::jsonb)
ON CONFLICT (tenant_id, event_id) DO NOTHING
RETURNING event_seq;
""";
private const string InsertDetailSql = """
INSERT INTO timeline.timeline_event_details
(event_id, tenant_id, envelope_version, raw_payload, normalized_payload)
VALUES
(@event_id, @tenant_id, @envelope_version, @raw_payload::jsonb, @normalized_payload::jsonb)
ON CONFLICT (event_id, tenant_id) DO NOTHING;
""";
private const string InsertDigestSql = """
INSERT INTO timeline.timeline_event_digests
(tenant_id, event_id, bundle_id, bundle_digest, attestation_subject, attestation_digest, manifest_uri)
VALUES
(@tenant_id, @event_id, @bundle_id, @bundle_digest, @attestation_subject, @attestation_digest, @manifest_uri)
ON CONFLICT (event_id, tenant_id) DO NOTHING;
""";
public async Task<bool> InsertAsync(TimelineEventEnvelope envelope, CancellationToken cancellationToken = default)
{
await using var connection = await DataSource.OpenConnectionAsync(envelope.TenantId, "writer", cancellationToken)
.ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
var inserted = await InsertEventAsync(connection, envelope, cancellationToken).ConfigureAwait(false);
if (!inserted)
{
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
return false;
}
await InsertDetailAsync(connection, envelope, cancellationToken).ConfigureAwait(false);
await InsertDigestAsync(connection, envelope, cancellationToken).ConfigureAwait(false);
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
return true;
}
private async Task<bool> InsertEventAsync(NpgsqlConnection connection, TimelineEventEnvelope envelope, CancellationToken cancellationToken)
{
await using var command = CreateCommand(InsertEventSql, connection);
AddParameter(command, "event_id", envelope.EventId);
AddParameter(command, "tenant_id", envelope.TenantId);
AddParameter(command, "source", envelope.Source);
AddParameter(command, "event_type", envelope.EventType);
AddParameter(command, "occurred_at", envelope.OccurredAt);
AddParameter(command, "correlation_id", envelope.CorrelationId);
AddParameter(command, "trace_id", envelope.TraceId);
AddParameter(command, "actor", envelope.Actor);
AddParameter(command, "severity", envelope.Severity);
AddParameter(command, "payload_hash", envelope.PayloadHash);
AddJsonbParameter(command, "attributes", envelope.Attributes is null
? "{}"
: JsonSerializer.Serialize(envelope.Attributes));
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return result is not null;
}
private async Task InsertDetailAsync(NpgsqlConnection connection, TimelineEventEnvelope envelope, CancellationToken cancellationToken)
{
await using var command = CreateCommand(InsertDetailSql, connection);
AddParameter(command, "event_id", envelope.EventId);
AddParameter(command, "tenant_id", envelope.TenantId);
AddParameter(command, "envelope_version", "orch.event.v1");
AddJsonbParameter(command, "raw_payload", envelope.RawPayloadJson);
AddJsonbParameter(command, "normalized_payload", envelope.NormalizedPayloadJson);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private async Task InsertDigestAsync(NpgsqlConnection connection, TimelineEventEnvelope envelope, CancellationToken cancellationToken)
{
if (envelope.BundleDigest is null && envelope.AttestationDigest is null && envelope.ManifestUri is null && envelope.BundleId is null)
{
return;
}
await using var command = CreateCommand(InsertDigestSql, connection);
AddParameter(command, "tenant_id", envelope.TenantId);
AddParameter(command, "event_id", envelope.EventId);
AddParameter(command, "bundle_id", envelope.BundleId);
AddParameter(command, "bundle_digest", envelope.BundleDigest);
AddParameter(command, "attestation_subject", envelope.AttestationSubject);
AddParameter(command, "attestation_digest", envelope.AttestationDigest);
AddParameter(command, "manifest_uri", envelope.ManifestUri);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
}

47
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Db/TimelineIndexerMigrationRunner.cs Normal file
View File

@@ -0,0 +1,47 @@
using System.Reflection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Infrastructure.Postgres.Migrations;
using StellaOps.Infrastructure.Postgres.Options;
namespace StellaOps.TimelineIndexer.Infrastructure.Db;
/// <summary>
/// Runs embedded SQL migrations for the Timeline Indexer schema.
/// </summary>
public sealed class TimelineIndexerMigrationRunner
{
private readonly PostgresOptions _options;
private readonly ILogger<TimelineIndexerMigrationRunner> _logger;
private const string ResourcePrefix = "StellaOps.TimelineIndexer.Infrastructure.Db.Migrations";
public TimelineIndexerMigrationRunner(
IOptions<PostgresOptions> options,
ILogger<TimelineIndexerMigrationRunner> logger)
{
_options = options.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>
/// Apply all pending migrations from embedded resources.
/// </summary>
public Task<int> RunAsync(CancellationToken cancellationToken = default)
{
var schema = string.IsNullOrWhiteSpace(_options.SchemaName)
? TimelineIndexerDataSource.DefaultSchemaName
: _options.SchemaName!;
var runner = new MigrationRunner(
_options.ConnectionString,
schema,
moduleName: "TimelineIndexer",
_logger);
return runner.RunFromAssemblyAsync(
assembly: Assembly.GetExecutingAssembly(),
resourcePrefix: ResourcePrefix,
cancellationToken);
}
}

85
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Db/TimelineQueryStore.cs Normal file
View File

@@ -0,0 +1,85 @@
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Infrastructure.Db;
public sealed class TimelineQueryStore(TimelineIndexerDataSource dataSource, ILogger<TimelineQueryStore> logger)
: RepositoryBase<TimelineIndexerDataSource>(dataSource, logger), ITimelineQueryStore
{
private const string BaseSelect = """
SELECT event_seq, event_id, tenant_id, event_type, source, occurred_at, received_at, correlation_id, trace_id, actor, severity, payload_hash
FROM timeline.timeline_events
WHERE tenant_id = @tenant_id
""";
public async Task<IReadOnlyList<TimelineEventView>> QueryAsync(string tenantId, TimelineQueryOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
var sql = new System.Text.StringBuilder(BaseSelect);
if (!string.IsNullOrWhiteSpace(options.EventType)) sql.Append(" AND event_type = @event_type");
if (!string.IsNullOrWhiteSpace(options.CorrelationId)) sql.Append(" AND correlation_id = @correlation_id");
if (!string.IsNullOrWhiteSpace(options.TraceId)) sql.Append(" AND trace_id = @trace_id");
if (!string.IsNullOrWhiteSpace(options.Severity)) sql.Append(" AND severity = @severity");
if (options.Since is not null) sql.Append(" AND occurred_at >= @since");
if (options.AfterEventSeq is not null) sql.Append(" AND event_seq < @after_seq");
sql.Append(" ORDER BY occurred_at DESC, event_seq DESC LIMIT @limit");
return await QueryAsync(
tenantId,
sql.ToString(),
cmd =>
{
AddParameter(cmd, "tenant_id", tenantId);
AddParameter(cmd, "event_type", options.EventType);
AddParameter(cmd, "correlation_id", options.CorrelationId);
AddParameter(cmd, "trace_id", options.TraceId);
AddParameter(cmd, "severity", options.Severity);
AddParameter(cmd, "since", options.Since);
AddParameter(cmd, "after_seq", options.AfterEventSeq);
AddParameter(cmd, "limit", Math.Clamp(options.Limit, 1, 500));
},
MapEvent,
cancellationToken).ConfigureAwait(false);
}
public async Task<TimelineEventView?> GetAsync(string tenantId, string eventId, CancellationToken cancellationToken)
{
const string sql = """
SELECT event_seq, event_id, tenant_id, event_type, source, occurred_at, received_at, correlation_id, trace_id, actor, severity, payload_hash
FROM timeline.timeline_events
WHERE tenant_id = @tenant_id AND event_id = @event_id
""";
return await QuerySingleOrDefaultAsync(
tenantId,
sql,
cmd =>
{
AddParameter(cmd, "tenant_id", tenantId);
AddParameter(cmd, "event_id", eventId);
},
MapEvent,
cancellationToken).ConfigureAwait(false);
}
private static TimelineEventView MapEvent(NpgsqlDataReader reader) => new()
{
EventSeq = reader.GetInt64(0),
EventId = reader.GetString(1),
TenantId = reader.GetString(2),
EventType = reader.GetString(3),
Source = reader.GetString(4),
OccurredAt = reader.GetFieldValue<DateTimeOffset>(5),
ReceivedAt = reader.GetFieldValue<DateTimeOffset>(6),
CorrelationId = GetNullableString(reader, 7),
TraceId = GetNullableString(reader, 8),
Actor = GetNullableString(reader, 9),
Severity = reader.GetString(10),
PayloadHash = GetNullableString(reader, 11)
};
}

30
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs Normal file
View File

@@ -0,0 +1,30 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Infrastructure.Postgres.Options;
using StellaOps.TimelineIndexer.Infrastructure.Db;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Services;
namespace StellaOps.TimelineIndexer.Infrastructure.DependencyInjection;
/// <summary>
/// Timeline Indexer PostgreSQL service registration helpers.
/// </summary>
public static class ServiceCollectionExtensions
{
private const string DefaultSection = "Postgres:Timeline";
/// <summary>
/// Registers Postgres options, data source, and migration runner for the Timeline Indexer.
/// </summary>
public static IServiceCollection AddTimelineIndexerPostgres(
this IServiceCollection services,
IConfiguration configuration,
string sectionName = DefaultSection)
{
services.Configure<PostgresOptions>(configuration.GetSection(sectionName));
services.AddSingleton<TimelineIndexerDataSource>();
services.AddSingleton<TimelineIndexerMigrationRunner>();
services.AddScoped<ITimelineEventStore, TimelineEventStore>();
services.AddScoped<ITimelineIngestionService, TimelineIngestionService>();
services.AddScoped<ITimel

48
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/StellaOps.TimelineIndexer.Infrastructure.csproj
View File

@@ -3,26 +3,38 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Core\StellaOps.TimelineIndexer.Core.csproj"/>
</ItemGroup>
<PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Core\StellaOps.TimelineIndexer.Core.csproj"/>
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.Postgres\StellaOps.Infrastructure.Postgres.csproj"/>
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
</Project>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<EmbeddedResource Include="Db/Migrations/*.sql" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Npgsql" Version="9.0.2" />
</ItemGroup>
</Project>

16
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Subscriptions/NullTimelineEventSubscriber.cs Normal file
View File

@@ -0,0 +1,16 @@
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Infrastructure.Subscriptions;
/// <summary>
/// Default no-op subscriber used until transport bindings are configured.
/// Keeps the ingestion worker running without requiring live brokers.
/// </summary>
public sealed class NullTimelineEventSubscriber : ITimelineEventSubscriber
{
public IAsyncEnumerable<TimelineEventEnvelope> SubscribeAsync(CancellationToken cancellationToken = default)
{
return AsyncEnumerable.Empty<TimelineEventEnvelope>();
}
}

32
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/TimelineIndexerDataSource.cs Normal file
View File

@@ -0,0 +1,32 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Infrastructure.Postgres.Connections;
using StellaOps.Infrastructure.Postgres.Options;
namespace StellaOps.TimelineIndexer.Infrastructure;
/// <summary>
/// PostgreSQL data source for the Timeline Indexer module.
/// Sets the default schema and carries tenant context via app.current_tenant.
/// </summary>
public sealed class TimelineIndexerDataSource : DataSourceBase
{
public const string DefaultSchemaName = "timeline";
public TimelineIndexerDataSource(IOptions<PostgresOptions> options, ILogger<TimelineIndexerDataSource> logger)
: base(EnsureSchema(options.Value), logger)
{
}
protected override string ModuleName => "TimelineIndexer";
private static PostgresOptions EnsureSchema(PostgresOptions baseOptions)
{
if (string.IsNullOrWhiteSpace(baseOptions.SchemaName))
{
baseOptions.SchemaName = DefaultSchemaName;
}
return baseOptions;
}
}

41
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Tests/InMemoryTimelineEventSubscriber.cs Normal file
View File

@@ -0,0 +1,41 @@
using System.Collections.Concurrent;
using System.Threading.Channels;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Tests;
public sealed class InMemoryTimelineEventSubscriber : ITimelineEventSubscriber
{
private readonly Channel<TimelineEventEnvelope> _channel;
public InMemoryTimelineEventSubscriber(IEnumerable<TimelineEventEnvelope>? seed = null)
{
_channel = Channel.CreateUnbounded<TimelineEventEnvelope>(new UnboundedChannelOptions
{
SingleReader = false,
SingleWriter = false
});
if (seed is not null)
{
foreach (var envelope in seed)
{
_channel.Writer.TryWrite(envelope);
}
_channel.Writer.TryComplete();
}
}
public void Enqueue(TimelineEventEnvelope envelope)
{
_channel.Writer.TryWrite(envelope);
}
public void Complete() => _channel.Writer.TryComplete();
public IAsyncEnumerable<TimelineEventEnvelope> SubscribeAsync(CancellationToken cancellationToken = default)
{
return _channel.Reader.ReadAllAsync(cancellationToken);
}
}

44
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Tests/StellaOps.TimelineIndexer.Tests.csproj
View File

@@ -53,11 +53,11 @@
<ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1"/>
@@ -111,25 +111,15 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Core\StellaOps.TimelineIndexer.Core.csproj"/>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Infrastructure\StellaOps.TimelineIndexer.Infrastructure.csproj"/>
</ItemGroup>
</Project>
<ItemGroup>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Core\StellaOps.TimelineIndexer.Core.csproj"/>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Infrastructure\StellaOps.TimelineIndexer.Infrastructure.csproj"/>
<ProjectReference Include="..\StellaOps.TimelineIndexer.Worker\StellaOps.TimelineIndexer.Worker.csproj"/>
</ItemGroup>
<ItemGroup>
<Using Include="System"/>
<Using Include="System.Threading.Tasks"/>
</ItemGroup>
</Project>

65
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Tests/TimelineIngestionServiceTests.cs Normal file
View File

@@ -0,0 +1,65 @@
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
using StellaOps.TimelineIndexer.Core.Services;
namespace StellaOps.TimelineIndexer.Tests;
public class TimelineIngestionServiceTests
{
[Fact]
public async Task Ingest_ComputesHash_WhenMissing()
{
var store = new FakeStore();
var service = new TimelineIngestionService(store);
var envelope = new TimelineEventEnvelope
{
EventId = "evt-1",
TenantId = "tenant-a",
EventType = "job.completed",
Source = "orchestrator",
OccurredAt = DateTimeOffset.UtcNow,
RawPayloadJson = """{"ok":true}"""
};
var result = await service.IngestAsync(envelope);
Assert.True(result.Inserted);
Assert.Equal("sha256:8ceeb2a3cfdd5c6c0257df04e3d6b7c29c6a54f9b89e0ee1d3f3f94a639a6a39", store.LastEnvelope?.PayloadHash);
}
[Fact]
public async Task Ingest_IsIdempotent_OnSameEventId()
{
var store = new FakeStore();
var service = new TimelineIngestionService(store);
var envelope = new TimelineEventEnvelope
{
EventId = "evt-dup",
TenantId = "tenant-a",
EventType = "job.completed",
Source = "orchestrator",
OccurredAt = DateTimeOffset.UtcNow,
RawPayloadJson = "{}"
};
var first = await service.IngestAsync(envelope);
var second = await service.IngestAsync(envelope);
Assert.True(first.Inserted);
Assert.False(second.Inserted);
}
private sealed class FakeStore : ITimelineEventStore
{
private readonly HashSet<(string tenant, string id)> _seen = new();
public TimelineEventEnvelope? LastEnvelope { get; private set; }
public Task<bool> InsertAsync(TimelineEventEnvelope envelope, CancellationToken cancellationToken = default)
{
LastEnvelope = envelope;
var key = (envelope.TenantId, envelope.EventId);
var inserted = _seen.Add(key);
return Task.FromResult(inserted);
}
}
}

65
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Tests/TimelineIngestionWorkerTests.cs Normal file
View File

@@ -0,0 +1,65 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
using StellaOps.TimelineIndexer.Core.Models.Results;
using StellaOps.TimelineIndexer.Core.Services;
using StellaOps.TimelineIndexer.Worker;
namespace StellaOps.TimelineIndexer.Tests;
public sealed class TimelineIngestionWorkerTests
{
[Fact]
public async Task Worker_Ingests_And_Dedupes()
{
var subscriber = new InMemoryTimelineEventSubscriber();
var store = new RecordingStore();
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton<ITimelineEventSubscriber>(subscriber);
serviceCollection.AddSingleton<ITimelineEventStore>(store);
serviceCollection.AddSingleton<ITimelineIngestionService, TimelineIngestionService>();
serviceCollection.AddSingleton<IHostedService, TimelineIngestionWorker>();
serviceCollection.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
using var host = serviceCollection.BuildServiceProvider();
var hosted = host.GetRequiredService<IHostedService>();
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
await hosted.StartAsync(cts.Token);
var evt = new TimelineEventEnvelope
{
EventId = "evt-1",
TenantId = "tenant-a",
EventType = "test",
Source = "unit",
OccurredAt = DateTimeOffset.UtcNow,
RawPayloadJson = "{}"
};
subscriber.Enqueue(evt);
subscriber.Enqueue(evt); // duplicate
subscriber.Complete();
await Task.Delay(200, cts.Token);
await hosted.StopAsync(cts.Token);
Assert.Equal(1, store.InsertCalls); // duplicate dropped
Assert.Equal("sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", store.LastHash); // hash of "{}"
}
private sealed class RecordingStore : ITimelineEventStore
{
private readonly HashSet<(string tenant, string id)> _seen = new();
public int InsertCalls { get; private set; }
public string? LastHash { get; private set; }
public Task<bool> InsertAsync(TimelineEventEnvelope envelope, CancellationToken cancellationToken = default)
{
InsertCalls++;
LastHash = envelope.PayloadHash;
return Task.FromResult(_seen.Add((envelope.TenantId, envelope.EventId)));
}
}
}

82
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Tests/TimelineSchemaTests.cs Normal file
View File

@@ -0,0 +1,82 @@
using Xunit;
namespace StellaOps.TimelineIndexer.Tests;
public sealed class TimelineSchemaTests
{
private static string FindRepoRoot()
{
var dir = AppContext.BaseDirectory;
for (var i = 0; i < 10 && dir is not null; i++)
{
if (File.Exists(Path.Combine(dir, "StellaOps.sln")) ||
File.Exists(Path.Combine(dir, "Directory.Build.props")))
{
return dir;
}
dir = Directory.GetParent(dir)?.FullName;
}
throw new InvalidOperationException("Could not locate repository root from test base directory.");
}
private static string ReadMigrationSql()
{
var root = FindRepoRoot();
var path = Path.Combine(
root,
"src",
"TimelineIndexer",
"StellaOps.TimelineIndexer",
"StellaOps.TimelineIndexer.Infrastructure",
"Db",
"Migrations",
"001_initial_schema.sql");
if (!File.Exists(path))
{
throw new FileNotFoundException("Expected migration file was not found.", path);
}
return File.ReadAllText(path);
}
[Fact]
public void MigrationFile_Exists()
{
var root = FindRepoRoot();
var path = Path.Combine(
root,
"src",
"TimelineIndexer",
"StellaOps.TimelineIndexer",
"StellaOps.TimelineIndexer.Infrastructure",
"Db",
"Migrations",
"001_initial_schema.sql");
Assert.True(File.Exists(path), $"Migration script missing at {path}");
}
[Fact]
public void Migration_EnablesRlsPolicies()
{
var sql = ReadMigrationSql();
Assert.Contains("timeline_app.require_current_tenant", sql, StringComparison.Ordinal);
Assert.Contains("timeline_events_isolation", sql, StringComparison.Ordinal);
Assert.Contains("timeline_event_details_isolation", sql, StringComparison.Ordinal);
Assert.Contains("timeline_event_digests_isolation", sql, StringComparison.Ordinal);
Assert.Contains("ENABLE ROW LEVEL SECURITY", sql, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void Migration_DefinesUniqueEventConstraint()
{
var sql = ReadMigrationSql();
Assert.Contains("UNIQUE (tenant_id, event_id)", sql, StringComparison.Ordinal);
Assert.Contains("event_seq bigserial", sql, StringComparison.Ordinal);
}
}

25
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/Program.cs
View File

@@ -1,7 +1,18 @@
using StellaOps.TimelineIndexer.Worker;
var builder = Host.CreateApplicationBuilder(args);
builder.Services.AddHostedService<Worker>();
var host = builder.Build();
host.Run();
using Microsoft.Extensions.Configuration;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Infrastructure.DependencyInjection;
using StellaOps.TimelineIndexer.Infrastructure.Subscriptions;
using StellaOps.TimelineIndexer.Worker;
var builder = Host.CreateApplicationBuilder(args);
builder.Configuration.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
builder.Configuration.AddJsonFile("appsettings.Development.json", optional: true, reloadOnChange: true);
builder.Configuration.AddEnvironmentVariables(prefix: "TIMELINE_");
builder.Services.AddTimelineIndexerPostgres(builder.Configuration);
builder.Services.AddSingleton<ITimelineEventSubscriber, NullTimelineEventSubscriber>();
builder.Services.AddHostedService<TimelineIngestionWorker>();
var host = builder.Build();
host.Run();

71
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/TimelineIngestionWorker.cs Normal file
View File

@@ -0,0 +1,71 @@
using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using System.Linq;
using StellaOps.TimelineIndexer.Core.Abstractions;
using StellaOps.TimelineIndexer.Core.Models;
namespace StellaOps.TimelineIndexer.Worker;
/// <summary>
/// Background consumer that reads timeline events from configured subscribers and persists them via the ingestion service.
/// </summary>
public sealed class TimelineIngestionWorker(
IEnumerable<ITimelineEventSubscriber> subscribers,
ITimelineIngestionService ingestionService,
ILogger<TimelineIngestionWorker> logger) : BackgroundService
{
private static readonly Meter Meter = new("StellaOps.TimelineIndexer", "1.0.0");
private static readonly Counter<long> IngestedCounter = Meter.CreateCounter<long>("timeline.ingested");
private static readonly Counter<long> DuplicateCounter = Meter.CreateCounter<long>("timeline.duplicates");
private static readonly Counter<long> FailedCounter = Meter.CreateCounter<long>("timeline.failed");
private readonly IEnumerable<ITimelineEventSubscriber> _subscribers = subscribers;
private readonly ITimelineIngestionService _ingestion = ingestionService;
private readonly ILogger<TimelineIngestionWorker> _logger = logger;
private readonly ConcurrentDictionary<(string tenant, string eventId), byte> _sessionSeen = new();
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
var tasks = _subscribers.Select(subscriber => ConsumeAsync(subscriber, stoppingToken)).ToArray();
return Task.WhenAll(tasks);
}
private async Task ConsumeAsync(ITimelineEventSubscriber subscriber, CancellationToken cancellationToken)
{
await foreach (var envelope in subscriber.SubscribeAsync(cancellationToken))
{
var key = (envelope.TenantId, envelope.EventId);
if (!_sessionSeen.TryAdd(key, 0))
{
DuplicateCounter.Add(1);
_logger.LogDebug("Skipped duplicate timeline event {EventId} for tenant {Tenant}", envelope.EventId, envelope.TenantId);
continue;
}
try
{
var result = await _ingestion.IngestAsync(envelope, cancellationToken).ConfigureAwait(false);
if (result.Inserted)
{
IngestedCounter.Add(1);
_logger.LogInformation("Ingested timeline event {EventId} from {Source} (tenant {Tenant})", envelope.EventId, envelope.Source, envelope.TenantId);
}
else
{
DuplicateCounter.Add(1);
_logger.LogDebug("Store reported duplicate for event {EventId} tenant {Tenant}", envelope.EventId, envelope.TenantId);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Respect shutdown.
break;
}
catch (Exception ex)
{
FailedCounter.Add(1);
_logger.LogError(ex, "Failed to ingest timeline event {EventId} for tenant {Tenant}", envelope.EventId, envelope.TenantId);
}
}
}
}

16
src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/Worker.cs
View File

@@ -1,16 +0,0 @@
namespace StellaOps.TimelineIndexer.Worker;
public class Worker(ILogger<Worker> logger) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
}
await Task.Delay(1000, stoppingToken);
}
}
}