stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Docs CI / lint-and-preview (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Details
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Details
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Details
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Details
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Details
devportal-offline / build-offline (push) Has been cancelled
Details
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 18:08:55 +02:00
parent 6e45066e37
commit f1a39c4ce3
234 changed files with 24038 additions and 6910 deletions
Download Patch File Download Diff File Expand all files Collapse all files

338
src/Excititor/__Libraries/StellaOps.Excititor.Storage.Postgres/Repositories/PostgresVexAttestationStore.cs Normal file
View File

@@ -0,0 +1,338 @@
using System.Collections.Immutable;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Excititor.Core.Evidence;
using StellaOps.Infrastructure.Postgres.Repositories;
namespace StellaOps.Excititor.Storage.Postgres.Repositories;
/// <summary>
/// PostgreSQL-backed store for VEX attestations.
/// </summary>
public sealed class PostgresVexAttestationStore : RepositoryBase<ExcititorDataSource>, IVexAttestationStore
{
private volatile bool _initialized;
private readonly SemaphoreSlim _initLock = new(1, 1);
public PostgresVexAttestationStore(ExcititorDataSource dataSource, ILogger<PostgresVexAttestationStore> logger)
: base(dataSource, logger)
{
}
public async ValueTask SaveAsync(VexStoredAttestation attestation, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(attestation);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
INSERT INTO vex.attestations (
attestation_id, tenant, manifest_id, merkle_root, dsse_envelope_json,
dsse_envelope_hash, item_count, attested_at, metadata
)
VALUES (
@attestation_id, @tenant, @manifest_id, @merkle_root, @dsse_envelope_json,
@dsse_envelope_hash, @item_count, @attested_at, @metadata
)
ON CONFLICT (tenant, attestation_id) DO UPDATE SET
manifest_id = EXCLUDED.manifest_id,
merkle_root = EXCLUDED.merkle_root,
dsse_envelope_json = EXCLUDED.dsse_envelope_json,
dsse_envelope_hash = EXCLUDED.dsse_envelope_hash,
item_count = EXCLUDED.item_count,
attested_at = EXCLUDED.attested_at,
metadata = EXCLUDED.metadata;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "attestation_id", attestation.AttestationId);
AddParameter(command, "tenant", attestation.Tenant);
AddParameter(command, "manifest_id", attestation.ManifestId);
AddParameter(command, "merkle_root", attestation.MerkleRoot);
AddParameter(command, "dsse_envelope_json", attestation.DsseEnvelopeJson);
AddParameter(command, "dsse_envelope_hash", attestation.DsseEnvelopeHash);
AddParameter(command, "item_count", attestation.ItemCount);
AddParameter(command, "attested_at", attestation.AttestedAt);
AddJsonbParameter(command, "metadata", SerializeMetadata(attestation.Metadata));
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
public async ValueTask<VexStoredAttestation?> FindByIdAsync(string tenant, string attestationId, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(tenant) || string.IsNullOrWhiteSpace(attestationId))
{
return null;
}
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT attestation_id, tenant, manifest_id, merkle_root, dsse_envelope_json,
dsse_envelope_hash, item_count, attested_at, metadata
FROM vex.attestations
WHERE LOWER(tenant) = LOWER(@tenant) AND attestation_id = @attestation_id;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant.Trim());
AddParameter(command, "attestation_id", attestationId.Trim());
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
return null;
}
return Map(reader);
}
public async ValueTask<VexStoredAttestation?> FindByManifestIdAsync(string tenant, string manifestId, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(tenant) || string.IsNullOrWhiteSpace(manifestId))
{
return null;
}
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT attestation_id, tenant, manifest_id, merkle_root, dsse_envelope_json,
dsse_envelope_hash, item_count, attested_at, metadata
FROM vex.attestations
WHERE LOWER(tenant) = LOWER(@tenant) AND LOWER(manifest_id) = LOWER(@manifest_id)
ORDER BY attested_at DESC
LIMIT 1;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant.Trim());
AddParameter(command, "manifest_id", manifestId.Trim());
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
return null;
}
return Map(reader);
}
public async ValueTask<VexAttestationListResult> ListAsync(VexAttestationQuery query, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(query);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
// Get total count
var countSql = "SELECT COUNT(*) FROM vex.attestations WHERE LOWER(tenant) = LOWER(@tenant)";
var whereClauses = new List<string>();
if (query.Since.HasValue)
{
whereClauses.Add("attested_at >= @since");
}
if (query.Until.HasValue)
{
whereClauses.Add("attested_at <= @until");
}
if (whereClauses.Count > 0)
{
countSql += " AND " + string.Join(" AND ", whereClauses);
}
await using var countCommand = CreateCommand(countSql, connection);
AddParameter(countCommand, "tenant", query.Tenant);
if (query.Since.HasValue)
{
AddParameter(countCommand, "since", query.Since.Value);
}
if (query.Until.HasValue)
{
AddParameter(countCommand, "until", query.Until.Value);
}
var totalCount = Convert.ToInt32(await countCommand.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false));
// Get items
var selectSql = """
SELECT attestation_id, tenant, manifest_id, merkle_root, dsse_envelope_json,
dsse_envelope_hash, item_count, attested_at, metadata
FROM vex.attestations
WHERE LOWER(tenant) = LOWER(@tenant)
""";
if (whereClauses.Count > 0)
{
selectSql += " AND " + string.Join(" AND ", whereClauses);
}
selectSql += " ORDER BY attested_at DESC, attestation_id ASC LIMIT @limit OFFSET @offset;";
await using var selectCommand = CreateCommand(selectSql, connection);
AddParameter(selectCommand, "tenant", query.Tenant);
AddParameter(selectCommand, "limit", query.Limit);
AddParameter(selectCommand, "offset", query.Offset);
if (query.Since.HasValue)
{
AddParameter(selectCommand, "since", query.Since.Value);
}
if (query.Until.HasValue)
{
AddParameter(selectCommand, "until", query.Until.Value);
}
var items = new List<VexStoredAttestation>();
await using var reader = await selectCommand.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
items.Add(Map(reader));
}
var hasMore = query.Offset + items.Count < totalCount;
return new VexAttestationListResult(items, totalCount, hasMore);
}
public async ValueTask<int> CountAsync(string tenant, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(tenant))
{
return 0;
}
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = "SELECT COUNT(*) FROM vex.attestations WHERE LOWER(tenant) = LOWER(@tenant);";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant.Trim());
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt32(result);
}
private static VexStoredAttestation Map(NpgsqlDataReader reader)
{
var attestationId = reader.GetString(0);
var tenant = reader.GetString(1);
var manifestId = reader.GetString(2);
var merkleRoot = reader.GetString(3);
var dsseEnvelopeJson = reader.GetString(4);
var dsseEnvelopeHash = reader.GetString(5);
var itemCount = reader.GetInt32(6);
var attestedAt = reader.GetFieldValue<DateTimeOffset>(7);
var metadataJson = reader.IsDBNull(8) ? null : reader.GetFieldValue<string>(8);
var metadata = DeserializeMetadata(metadataJson);
return new VexStoredAttestation(
attestationId,
tenant,
manifestId,
merkleRoot,
dsseEnvelopeJson,
dsseEnvelopeHash,
itemCount,
attestedAt,
metadata);
}
private static string SerializeMetadata(ImmutableDictionary<string, string> metadata)
{
if (metadata.IsEmpty)
{
return "{}";
}
return JsonSerializer.Serialize(metadata);
}
private static ImmutableDictionary<string, string> DeserializeMetadata(string? json)
{
if (string.IsNullOrWhiteSpace(json))
{
return ImmutableDictionary<string, string>.Empty;
}
try
{
using var doc = JsonDocument.Parse(json);
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var property in doc.RootElement.EnumerateObject())
{
if (property.Value.ValueKind == JsonValueKind.String)
{
var value = property.Value.GetString();
if (value is not null)
{
builder[property.Name] = value;
}
}
}
return builder.ToImmutable();
}
catch
{
return ImmutableDictionary<string, string>.Empty;
}
}
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
{
if (_initialized)
{
return;
}
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_initialized)
{
return;
}
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
const string sql = """
CREATE TABLE IF NOT EXISTS vex.attestations (
attestation_id TEXT NOT NULL,
tenant TEXT NOT NULL,
manifest_id TEXT NOT NULL,
merkle_root TEXT NOT NULL,
dsse_envelope_json TEXT NOT NULL,
dsse_envelope_hash TEXT NOT NULL,
item_count INTEGER NOT NULL,
attested_at TIMESTAMPTZ NOT NULL,
metadata JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (tenant, attestation_id)
);
CREATE INDEX IF NOT EXISTS idx_attestations_tenant ON vex.attestations(tenant);
CREATE INDEX IF NOT EXISTS idx_attestations_manifest_id ON vex.attestations(tenant, manifest_id);
CREATE INDEX IF NOT EXISTS idx_attestations_attested_at ON vex.attestations(tenant, attested_at DESC);
""";
await using var command = CreateCommand(sql, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_initialized = true;
}
finally
{
_initLock.Release();
}
}
}

700
src/Excititor/__Libraries/StellaOps.Excititor.Storage.Postgres/Repositories/PostgresVexObservationStore.cs Normal file
View File

@@ -0,0 +1,700 @@
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.Json.Nodes;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Infrastructure.Postgres.Repositories;
namespace StellaOps.Excititor.Storage.Postgres.Repositories;
/// <summary>
/// PostgreSQL-backed store for VEX observations with complex nested structures.
/// </summary>
public sealed class PostgresVexObservationStore : RepositoryBase<ExcititorDataSource>, IVexObservationStore
{
private volatile bool _initialized;
private readonly SemaphoreSlim _initLock = new(1, 1);
public PostgresVexObservationStore(ExcititorDataSource dataSource, ILogger<PostgresVexObservationStore> logger)
: base(dataSource, logger)
{
}
public async ValueTask<bool> InsertAsync(VexObservation observation, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(observation);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
INSERT INTO vex.observations (
observation_id, tenant, provider_id, stream_id, upstream, statements,
content, linkset, created_at, supersedes, attributes
)
VALUES (
@observation_id, @tenant, @provider_id, @stream_id, @upstream, @statements,
@content, @linkset, @created_at, @supersedes, @attributes
)
ON CONFLICT (tenant, observation_id) DO NOTHING;
""";
await using var command = CreateCommand(sql, connection);
AddObservationParameters(command, observation);
var affected = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
return affected > 0;
}
public async ValueTask<bool> UpsertAsync(VexObservation observation, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(observation);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
INSERT INTO vex.observations (
observation_id, tenant, provider_id, stream_id, upstream, statements,
content, linkset, created_at, supersedes, attributes
)
VALUES (
@observation_id, @tenant, @provider_id, @stream_id, @upstream, @statements,
@content, @linkset, @created_at, @supersedes, @attributes
)
ON CONFLICT (tenant, observation_id) DO UPDATE SET
provider_id = EXCLUDED.provider_id,
stream_id = EXCLUDED.stream_id,
upstream = EXCLUDED.upstream,
statements = EXCLUDED.statements,
content = EXCLUDED.content,
linkset = EXCLUDED.linkset,
created_at = EXCLUDED.created_at,
supersedes = EXCLUDED.supersedes,
attributes = EXCLUDED.attributes;
""";
await using var command = CreateCommand(sql, connection);
AddObservationParameters(command, observation);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
return true;
}
public async ValueTask<int> InsertManyAsync(string tenant, IEnumerable<VexObservation> observations, CancellationToken cancellationToken)
{
if (observations is null)
{
return 0;
}
var observationsList = observations
.Where(o => string.Equals(o.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.ToList();
if (observationsList.Count == 0)
{
return 0;
}
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
var count = 0;
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
foreach (var observation in observationsList)
{
const string sql = """
INSERT INTO vex.observations (
observation_id, tenant, provider_id, stream_id, upstream, statements,
content, linkset, created_at, supersedes, attributes
)
VALUES (
@observation_id, @tenant, @provider_id, @stream_id, @upstream, @statements,
@content, @linkset, @created_at, @supersedes, @attributes
)
ON CONFLICT (tenant, observation_id) DO NOTHING;
""";
await using var command = CreateCommand(sql, connection);
AddObservationParameters(command, observation);
var affected = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
if (affected > 0)
{
count++;
}
}
return count;
}
public async ValueTask<VexObservation?> GetByIdAsync(string tenant, string observationId, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(tenant) || string.IsNullOrWhiteSpace(observationId))
{
return null;
}
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT observation_id, tenant, provider_id, stream_id, upstream, statements,
content, linkset, created_at, supersedes, attributes
FROM vex.observations
WHERE LOWER(tenant) = LOWER(@tenant) AND observation_id = @observation_id;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant.Trim());
AddParameter(command, "observation_id", observationId.Trim());
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
return null;
}
return Map(reader);
}
public async ValueTask<IReadOnlyList<VexObservation>> FindByVulnerabilityAndProductAsync(
string tenant,
string vulnerabilityId,
string productKey,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
// Use JSONB containment to query nested statements array
const string sql = """
SELECT observation_id, tenant, provider_id, stream_id, upstream, statements,
content, linkset, created_at, supersedes, attributes
FROM vex.observations
WHERE LOWER(tenant) = LOWER(@tenant)
AND EXISTS (
SELECT 1 FROM jsonb_array_elements(statements) AS stmt
WHERE LOWER(stmt->>'vulnerabilityId') = LOWER(@vulnerability_id)
AND LOWER(stmt->>'productKey') = LOWER(@product_key)
)
ORDER BY created_at DESC;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "vulnerability_id", vulnerabilityId);
AddParameter(command, "product_key", productKey);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyList<VexObservation>> FindByProviderAsync(
string tenant,
string providerId,
int limit,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT observation_id, tenant, provider_id, stream_id, upstream, statements,
content, linkset, created_at, supersedes, attributes
FROM vex.observations
WHERE LOWER(tenant) = LOWER(@tenant) AND LOWER(provider_id) = LOWER(@provider_id)
ORDER BY created_at DESC
LIMIT @limit;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "provider_id", providerId);
AddParameter(command, "limit", limit);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<bool> DeleteAsync(string tenant, string observationId, CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
DELETE FROM vex.observations
WHERE LOWER(tenant) = LOWER(@tenant) AND observation_id = @observation_id;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "observation_id", observationId);
var affected = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
return affected > 0;
}
public async ValueTask<long> CountAsync(string tenant, CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = "SELECT COUNT(*) FROM vex.observations WHERE LOWER(tenant) = LOWER(@tenant);";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt64(result);
}
private void AddObservationParameters(NpgsqlCommand command, VexObservation observation)
{
AddParameter(command, "observation_id", observation.ObservationId);
AddParameter(command, "tenant", observation.Tenant);
AddParameter(command, "provider_id", observation.ProviderId);
AddParameter(command, "stream_id", observation.StreamId);
AddJsonbParameter(command, "upstream", SerializeUpstream(observation.Upstream));
AddJsonbParameter(command, "statements", SerializeStatements(observation.Statements));
AddJsonbParameter(command, "content", SerializeContent(observation.Content));
AddJsonbParameter(command, "linkset", SerializeLinkset(observation.Linkset));
AddParameter(command, "created_at", observation.CreatedAt);
AddParameter(command, "supersedes", observation.Supersedes.IsDefaultOrEmpty ? Array.Empty<string>() : observation.Supersedes.ToArray());
AddJsonbParameter(command, "attributes", SerializeAttributes(observation.Attributes));
}
private static async Task<IReadOnlyList<VexObservation>> ExecuteQueryAsync(NpgsqlCommand command, CancellationToken cancellationToken)
{
var results = new List<VexObservation>();
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
results.Add(Map(reader));
}
return results;
}
private static VexObservation Map(NpgsqlDataReader reader)
{
var observationId = reader.GetString(0);
var tenant = reader.GetString(1);
var providerId = reader.GetString(2);
var streamId = reader.GetString(3);
var upstreamJson = reader.GetFieldValue<string>(4);
var statementsJson = reader.GetFieldValue<string>(5);
var contentJson = reader.GetFieldValue<string>(6);
var linksetJson = reader.GetFieldValue<string>(7);
var createdAt = reader.GetFieldValue<DateTimeOffset>(8);
var supersedes = reader.IsDBNull(9) ? Array.Empty<string>() : reader.GetFieldValue<string[]>(9);
var attributesJson = reader.IsDBNull(10) ? null : reader.GetFieldValue<string>(10);
var upstream = DeserializeUpstream(upstreamJson);
var statements = DeserializeStatements(statementsJson);
var content = DeserializeContent(contentJson);
var linkset = DeserializeLinkset(linksetJson);
var attributes = DeserializeAttributes(attributesJson);
return new VexObservation(
observationId,
tenant,
providerId,
streamId,
upstream,
statements,
content,
linkset,
createdAt,
supersedes.Length == 0 ? null : supersedes.ToImmutableArray(),
attributes);
}
#region Serialization
private static string SerializeUpstream(VexObservationUpstream upstream)
{
var obj = new
{
upstreamId = upstream.UpstreamId,
documentVersion = upstream.DocumentVersion,
fetchedAt = upstream.FetchedAt,
receivedAt = upstream.ReceivedAt,
contentHash = upstream.ContentHash,
signature = new
{
present = upstream.Signature.Present,
format = upstream.Signature.Format,
keyId = upstream.Signature.KeyId,
signature = upstream.Signature.Signature
},
metadata = upstream.Metadata
};
return JsonSerializer.Serialize(obj);
}
private static VexObservationUpstream DeserializeUpstream(string json)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var upstreamId = root.GetProperty("upstreamId").GetString()!;
var documentVersion = root.TryGetProperty("documentVersion", out var dv) && dv.ValueKind == JsonValueKind.String
? dv.GetString()
: null;
var fetchedAt = root.GetProperty("fetchedAt").GetDateTimeOffset();
var receivedAt = root.GetProperty("receivedAt").GetDateTimeOffset();
var contentHash = root.GetProperty("contentHash").GetString()!;
var sigElem = root.GetProperty("signature");
var signature = new VexObservationSignature(
sigElem.GetProperty("present").GetBoolean(),
sigElem.TryGetProperty("format", out var f) && f.ValueKind == JsonValueKind.String ? f.GetString() : null,
sigElem.TryGetProperty("keyId", out var k) && k.ValueKind == JsonValueKind.String ? k.GetString() : null,
sigElem.TryGetProperty("signature", out var s) && s.ValueKind == JsonValueKind.String ? s.GetString() : null);
var metadata = DeserializeStringDict(root, "metadata");
return new VexObservationUpstream(upstreamId, documentVersion, fetchedAt, receivedAt, contentHash, signature, metadata);
}
private static string SerializeStatements(ImmutableArray<VexObservationStatement> statements)
{
var list = statements.Select(s => new
{
vulnerabilityId = s.VulnerabilityId,
productKey = s.ProductKey,
status = s.Status.ToString(),
lastObserved = s.LastObserved,
locator = s.Locator,
justification = s.Justification?.ToString(),
introducedVersion = s.IntroducedVersion,
fixedVersion = s.FixedVersion,
purl = s.Purl,
cpe = s.Cpe,
evidence = s.Evidence.Select(e => e?.ToJsonString()),
metadata = s.Metadata
}).ToArray();
return JsonSerializer.Serialize(list);
}
private static ImmutableArray<VexObservationStatement> DeserializeStatements(string json)
{
using var doc = JsonDocument.Parse(json);
var builder = ImmutableArray.CreateBuilder<VexObservationStatement>();
foreach (var elem in doc.RootElement.EnumerateArray())
{
var vulnId = elem.GetProperty("vulnerabilityId").GetString()!;
var productKey = elem.GetProperty("productKey").GetString()!;
var statusStr = elem.GetProperty("status").GetString()!;
var status = Enum.TryParse<VexClaimStatus>(statusStr, ignoreCase: true, out var st) ? st : VexClaimStatus.Affected;
DateTimeOffset? lastObserved = null;
if (elem.TryGetProperty("lastObserved", out var lo) && lo.ValueKind != JsonValueKind.Null)
{
lastObserved = lo.GetDateTimeOffset();
}
var locator = GetOptionalString(elem, "locator");
VexJustification? justification = null;
if (elem.TryGetProperty("justification", out var jElem) && jElem.ValueKind == JsonValueKind.String)
{
var justStr = jElem.GetString();
if (!string.IsNullOrWhiteSpace(justStr) && Enum.TryParse<VexJustification>(justStr, ignoreCase: true, out var j))
{
justification = j;
}
}
var introducedVersion = GetOptionalString(elem, "introducedVersion");
var fixedVersion = GetOptionalString(elem, "fixedVersion");
var purl = GetOptionalString(elem, "purl");
var cpe = GetOptionalString(elem, "cpe");
ImmutableArray<JsonNode>? evidence = null;
if (elem.TryGetProperty("evidence", out var evElem) && evElem.ValueKind == JsonValueKind.Array)
{
var evBuilder = ImmutableArray.CreateBuilder<JsonNode>();
foreach (var evItem in evElem.EnumerateArray())
{
if (evItem.ValueKind == JsonValueKind.String)
{
var evStr = evItem.GetString();
if (!string.IsNullOrWhiteSpace(evStr))
{
var node = JsonNode.Parse(evStr);
if (node is not null)
{
evBuilder.Add(node);
}
}
}
}
if (evBuilder.Count > 0)
{
evidence = evBuilder.ToImmutable();
}
}
var metadata = DeserializeStringDict(elem, "metadata");
builder.Add(new VexObservationStatement(
vulnId, productKey, status, lastObserved, locator, justification,
introducedVersion, fixedVersion, purl, cpe, evidence, metadata));
}
return builder.ToImmutable();
}
private static string SerializeContent(VexObservationContent content)
{
var obj = new
{
format = content.Format,
specVersion = content.SpecVersion,
raw = content.Raw.ToJsonString(),
metadata = content.Metadata
};
return JsonSerializer.Serialize(obj);
}
private static VexObservationContent DeserializeContent(string json)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var format = root.GetProperty("format").GetString()!;
var specVersion = GetOptionalString(root, "specVersion");
var rawStr = root.GetProperty("raw").GetString()!;
var raw = JsonNode.Parse(rawStr)!;
var metadata = DeserializeStringDict(root, "metadata");
return new VexObservationContent(format, specVersion, raw, metadata);
}
private static string SerializeLinkset(VexObservationLinkset linkset)
{
var obj = new
{
aliases = linkset.Aliases.ToArray(),
purls = linkset.Purls.ToArray(),
cpes = linkset.Cpes.ToArray(),
references = linkset.References.Select(r => new { type = r.Type, url = r.Url }).ToArray(),
reconciledFrom = linkset.ReconciledFrom.ToArray(),
disagreements = linkset.Disagreements.Select(d => new
{
providerId = d.ProviderId,
status = d.Status,
justification = d.Justification,
confidence = d.Confidence
}).ToArray(),
observations = linkset.Observations.Select(o => new
{
observationId = o.ObservationId,
providerId = o.ProviderId,
status = o.Status,
confidence = o.Confidence
}).ToArray()
};
return JsonSerializer.Serialize(obj);
}
private static VexObservationLinkset DeserializeLinkset(string json)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var aliases = GetStringArray(root, "aliases");
var purls = GetStringArray(root, "purls");
var cpes = GetStringArray(root, "cpes");
var reconciledFrom = GetStringArray(root, "reconciledFrom");
var references = new List<VexObservationReference>();
if (root.TryGetProperty("references", out var refsElem) && refsElem.ValueKind == JsonValueKind.Array)
{
foreach (var refElem in refsElem.EnumerateArray())
{
var type = refElem.GetProperty("type").GetString()!;
var url = refElem.GetProperty("url").GetString()!;
references.Add(new VexObservationReference(type, url));
}
}
var disagreements = new List<VexObservationDisagreement>();
if (root.TryGetProperty("disagreements", out var disElem) && disElem.ValueKind == JsonValueKind.Array)
{
foreach (var dElem in disElem.EnumerateArray())
{
var providerId = dElem.GetProperty("providerId").GetString()!;
var status = dElem.GetProperty("status").GetString()!;
var justification = GetOptionalString(dElem, "justification");
double? confidence = null;
if (dElem.TryGetProperty("confidence", out var c) && c.ValueKind == JsonValueKind.Number)
{
confidence = c.GetDouble();
}
disagreements.Add(new VexObservationDisagreement(providerId, status, justification, confidence));
}
}
var observationRefs = new List<VexLinksetObservationRefModel>();
if (root.TryGetProperty("observations", out var obsElem) && obsElem.ValueKind == JsonValueKind.Array)
{
foreach (var oElem in obsElem.EnumerateArray())
{
var obsId = oElem.GetProperty("observationId").GetString()!;
var providerId = oElem.GetProperty("providerId").GetString()!;
var status = oElem.GetProperty("status").GetString()!;
double? confidence = null;
if (oElem.TryGetProperty("confidence", out var c) && c.ValueKind == JsonValueKind.Number)
{
confidence = c.GetDouble();
}
observationRefs.Add(new VexLinksetObservationRefModel(obsId, providerId, status, confidence));
}
}
return new VexObservationLinkset(aliases, purls, cpes, references, reconciledFrom, disagreements, observationRefs);
}
private static string SerializeAttributes(ImmutableDictionary<string, string> attributes)
{
if (attributes.IsEmpty)
{
return "{}";
}
return JsonSerializer.Serialize(attributes);
}
private static ImmutableDictionary<string, string> DeserializeAttributes(string? json)
{
if (string.IsNullOrWhiteSpace(json))
{
return ImmutableDictionary<string, string>.Empty;
}
try
{
using var doc = JsonDocument.Parse(json);
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var property in doc.RootElement.EnumerateObject())
{
if (property.Value.ValueKind == JsonValueKind.String)
{
var value = property.Value.GetString();
if (value is not null)
{
builder[property.Name] = value;
}
}
}
return builder.ToImmutable();
}
catch
{
return ImmutableDictionary<string, string>.Empty;
}
}
private static ImmutableDictionary<string, string> DeserializeStringDict(JsonElement elem, string propertyName)
{
if (!elem.TryGetProperty(propertyName, out var prop) || prop.ValueKind != JsonValueKind.Object)
{
return ImmutableDictionary<string, string>.Empty;
}
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var p in prop.EnumerateObject())
{
if (p.Value.ValueKind == JsonValueKind.String)
{
var val = p.Value.GetString();
if (val is not null)
{
builder[p.Name] = val;
}
}
}
return builder.ToImmutable();
}
private static string? GetOptionalString(JsonElement elem, string propertyName)
{
if (elem.TryGetProperty(propertyName, out var prop) && prop.ValueKind == JsonValueKind.String)
{
return prop.GetString();
}
return null;
}
private static IEnumerable<string> GetStringArray(JsonElement elem, string propertyName)
{
if (!elem.TryGetProperty(propertyName, out var prop) || prop.ValueKind != JsonValueKind.Array)
{
return Enumerable.Empty<string>();
}
return prop.EnumerateArray()
.Where(e => e.ValueKind == JsonValueKind.String)
.Select(e => e.GetString()!)
.Where(s => !string.IsNullOrWhiteSpace(s));
}
#endregion
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
{
if (_initialized)
{
return;
}
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_initialized)
{
return;
}
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
const string sql = """
CREATE TABLE IF NOT EXISTS vex.observations (
observation_id TEXT NOT NULL,
tenant TEXT NOT NULL,
provider_id TEXT NOT NULL,
stream_id TEXT NOT NULL,
upstream JSONB NOT NULL,
statements JSONB NOT NULL DEFAULT '[]',
content JSONB NOT NULL,
linkset JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL,
supersedes TEXT[] NOT NULL DEFAULT '{}',
attributes JSONB NOT NULL DEFAULT '{}',
PRIMARY KEY (tenant, observation_id)
);
CREATE INDEX IF NOT EXISTS idx_observations_tenant ON vex.observations(tenant);
CREATE INDEX IF NOT EXISTS idx_observations_provider ON vex.observations(tenant, provider_id);
CREATE INDEX IF NOT EXISTS idx_observations_created_at ON vex.observations(tenant, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_observations_statements ON vex.observations USING GIN (statements);
""";
await using var command = CreateCommand(sql, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_initialized = true;
}
finally
{
_initLock.Release();
}
}
}

268
src/Excititor/__Libraries/StellaOps.Excititor.Storage.Postgres/Repositories/PostgresVexProviderStore.cs Normal file
View File

@@ -0,0 +1,268 @@
using System.Collections.Immutable;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Infrastructure.Postgres.Repositories;
namespace StellaOps.Excititor.Storage.Postgres.Repositories;
/// <summary>
/// PostgreSQL-backed provider store for VEX provider registry.
/// </summary>
public sealed class PostgresVexProviderStore : RepositoryBase<ExcititorDataSource>, IVexProviderStore
{
private volatile bool _initialized;
private readonly SemaphoreSlim _initLock = new(1, 1);
public PostgresVexProviderStore(ExcititorDataSource dataSource, ILogger<PostgresVexProviderStore> logger)
: base(dataSource, logger)
{
}
public async ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(id);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT id, display_name, kind, base_uris, discovery, trust, enabled
FROM vex.providers
WHERE LOWER(id) = LOWER(@id);
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "id", id);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
return null;
}
return Map(reader);
}
public async ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(provider);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
INSERT INTO vex.providers (id, display_name, kind, base_uris, discovery, trust, enabled)
VALUES (@id, @display_name, @kind, @base_uris, @discovery, @trust, @enabled)
ON CONFLICT (id) DO UPDATE SET
display_name = EXCLUDED.display_name,
kind = EXCLUDED.kind,
base_uris = EXCLUDED.base_uris,
discovery = EXCLUDED.discovery,
trust = EXCLUDED.trust,
enabled = EXCLUDED.enabled;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "id", provider.Id);
AddParameter(command, "display_name", provider.DisplayName);
AddParameter(command, "kind", provider.Kind.ToString().ToLowerInvariant());
AddParameter(command, "base_uris", provider.BaseUris.IsDefault ? Array.Empty<string>() : provider.BaseUris.Select(u => u.ToString()).ToArray());
AddJsonbParameter(command, "discovery", SerializeDiscovery(provider.Discovery));
AddJsonbParameter(command, "trust", SerializeTrust(provider.Trust));
AddParameter(command, "enabled", provider.Enabled);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT id, display_name, kind, base_uris, discovery, trust, enabled
FROM vex.providers
ORDER BY id;
""";
await using var command = CreateCommand(sql, connection);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
var results = new List<VexProvider>();
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
results.Add(Map(reader));
}
return results;
}
private VexProvider Map(NpgsqlDataReader reader)
{
var id = reader.GetString(0);
var displayName = reader.GetString(1);
var kindStr = reader.GetString(2);
var baseUrisArr = reader.IsDBNull(3) ? Array.Empty<string>() : reader.GetFieldValue<string[]>(3);
var discoveryJson = reader.IsDBNull(4) ? null : reader.GetFieldValue<string>(4);
var trustJson = reader.IsDBNull(5) ? null : reader.GetFieldValue<string>(5);
var enabled = reader.IsDBNull(6) || reader.GetBoolean(6);
var kind = Enum.TryParse<VexProviderKind>(kindStr, ignoreCase: true, out var k) ? k : VexProviderKind.Vendor;
var baseUris = baseUrisArr.Select(s => new Uri(s)).ToArray();
var discovery = DeserializeDiscovery(discoveryJson);
var trust = DeserializeTrust(trustJson);
return new VexProvider(id, displayName, kind, baseUris, discovery, trust, enabled);
}
private static string SerializeDiscovery(VexProviderDiscovery discovery)
{
var obj = new
{
wellKnownMetadata = discovery.WellKnownMetadata?.ToString(),
rolieService = discovery.RolIeService?.ToString()
};
return JsonSerializer.Serialize(obj);
}
private static VexProviderDiscovery DeserializeDiscovery(string? json)
{
if (string.IsNullOrWhiteSpace(json))
{
return VexProviderDiscovery.Empty;
}
try
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
Uri? wellKnown = null;
Uri? rolie = null;
if (root.TryGetProperty("wellKnownMetadata", out var wkProp) && wkProp.ValueKind == JsonValueKind.String)
{
var wkStr = wkProp.GetString();
if (!string.IsNullOrWhiteSpace(wkStr))
{
wellKnown = new Uri(wkStr);
}
}
if (root.TryGetProperty("rolieService", out var rsProp) && rsProp.ValueKind == JsonValueKind.String)
{
var rsStr = rsProp.GetString();
if (!string.IsNullOrWhiteSpace(rsStr))
{
rolie = new Uri(rsStr);
}
}
return new VexProviderDiscovery(wellKnown, rolie);
}
catch
{
return VexProviderDiscovery.Empty;
}
}
private static string SerializeTrust(VexProviderTrust trust)
{
var obj = new
{
weight = trust.Weight,
cosign = trust.Cosign is null ? null : new { issuer = trust.Cosign.Issuer, identityPattern = trust.Cosign.IdentityPattern },
pgpFingerprints = trust.PgpFingerprints.IsDefault ? Array.Empty<string>() : trust.PgpFingerprints.ToArray()
};
return JsonSerializer.Serialize(obj);
}
private static VexProviderTrust DeserializeTrust(string? json)
{
if (string.IsNullOrWhiteSpace(json))
{
return VexProviderTrust.Default;
}
try
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var weight = 1.0;
if (root.TryGetProperty("weight", out var wProp) && wProp.TryGetDouble(out var w))
{
weight = w;
}
VexCosignTrust? cosign = null;
if (root.TryGetProperty("cosign", out var cProp) && cProp.ValueKind == JsonValueKind.Object)
{
var issuer = cProp.TryGetProperty("issuer", out var iProp) ? iProp.GetString() : null;
var pattern = cProp.TryGetProperty("identityPattern", out var pProp) ? pProp.GetString() : null;
if (!string.IsNullOrWhiteSpace(issuer) && !string.IsNullOrWhiteSpace(pattern))
{
cosign = new VexCosignTrust(issuer, pattern);
}
}
IEnumerable<string>? fingerprints = null;
if (root.TryGetProperty("pgpFingerprints", out var fProp) && fProp.ValueKind == JsonValueKind.Array)
{
fingerprints = fProp.EnumerateArray()
.Where(e => e.ValueKind == JsonValueKind.String)
.Select(e => e.GetString()!)
.Where(s => !string.IsNullOrWhiteSpace(s));
}
return new VexProviderTrust(weight, cosign, fingerprints);
}
catch
{
return VexProviderTrust.Default;
}
}
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
{
if (_initialized)
{
return;
}
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_initialized)
{
return;
}
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
const string sql = """
CREATE TABLE IF NOT EXISTS vex.providers (
id TEXT PRIMARY KEY,
display_name TEXT NOT NULL,
kind TEXT NOT NULL CHECK (kind IN ('vendor', 'distro', 'hub', 'platform', 'attestation')),
base_uris TEXT[] NOT NULL DEFAULT '{}',
discovery JSONB NOT NULL DEFAULT '{}',
trust JSONB NOT NULL DEFAULT '{}',
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_providers_kind ON vex.providers(kind);
CREATE INDEX IF NOT EXISTS idx_providers_enabled ON vex.providers(enabled) WHERE enabled = TRUE;
""";
await using var command = CreateCommand(sql, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_initialized = true;
}
finally
{
_initLock.Release();
}
}
}

442
src/Excititor/__Libraries/StellaOps.Excititor.Storage.Postgres/Repositories/PostgresVexTimelineEventStore.cs Normal file
View File

@@ -0,0 +1,442 @@
using System.Collections.Immutable;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Infrastructure.Postgres.Repositories;
namespace StellaOps.Excititor.Storage.Postgres.Repositories;
/// <summary>
/// PostgreSQL-backed store for VEX timeline events.
/// </summary>
public sealed class PostgresVexTimelineEventStore : RepositoryBase<ExcititorDataSource>, IVexTimelineEventStore
{
private volatile bool _initialized;
private readonly SemaphoreSlim _initLock = new(1, 1);
public PostgresVexTimelineEventStore(ExcititorDataSource dataSource, ILogger<PostgresVexTimelineEventStore> logger)
: base(dataSource, logger)
{
}
public async ValueTask<string> InsertAsync(TimelineEvent evt, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(evt);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
INSERT INTO vex.timeline_events (
event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
)
VALUES (
@event_id, @tenant, @provider_id, @stream_id, @event_type, @trace_id,
@justification_summary, @evidence_hash, @payload_hash, @created_at, @attributes
)
ON CONFLICT (tenant, event_id) DO NOTHING
RETURNING event_id;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "event_id", evt.EventId);
AddParameter(command, "tenant", evt.Tenant);
AddParameter(command, "provider_id", evt.ProviderId);
AddParameter(command, "stream_id", evt.StreamId);
AddParameter(command, "event_type", evt.EventType);
AddParameter(command, "trace_id", evt.TraceId);
AddParameter(command, "justification_summary", evt.JustificationSummary);
AddParameter(command, "evidence_hash", (object?)evt.EvidenceHash ?? DBNull.Value);
AddParameter(command, "payload_hash", (object?)evt.PayloadHash ?? DBNull.Value);
AddParameter(command, "created_at", evt.CreatedAt);
AddJsonbParameter(command, "attributes", SerializeAttributes(evt.Attributes));
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return result?.ToString() ?? evt.EventId;
}
public async ValueTask<int> InsertManyAsync(string tenant, IEnumerable<TimelineEvent> events, CancellationToken cancellationToken)
{
if (events is null)
{
return 0;
}
var eventsList = events.Where(e => string.Equals(e.Tenant, tenant, StringComparison.OrdinalIgnoreCase)).ToList();
if (eventsList.Count == 0)
{
return 0;
}
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
var count = 0;
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
foreach (var evt in eventsList)
{
const string sql = """
INSERT INTO vex.timeline_events (
event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
)
VALUES (
@event_id, @tenant, @provider_id, @stream_id, @event_type, @trace_id,
@justification_summary, @evidence_hash, @payload_hash, @created_at, @attributes
)
ON CONFLICT (tenant, event_id) DO NOTHING;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "event_id", evt.EventId);
AddParameter(command, "tenant", evt.Tenant);
AddParameter(command, "provider_id", evt.ProviderId);
AddParameter(command, "stream_id", evt.StreamId);
AddParameter(command, "event_type", evt.EventType);
AddParameter(command, "trace_id", evt.TraceId);
AddParameter(command, "justification_summary", evt.JustificationSummary);
AddParameter(command, "evidence_hash", (object?)evt.EvidenceHash ?? DBNull.Value);
AddParameter(command, "payload_hash", (object?)evt.PayloadHash ?? DBNull.Value);
AddParameter(command, "created_at", evt.CreatedAt);
AddJsonbParameter(command, "attributes", SerializeAttributes(evt.Attributes));
var affected = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
if (affected > 0)
{
count++;
}
}
return count;
}
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByTimeRangeAsync(
string tenant,
DateTimeOffset from,
DateTimeOffset to,
int limit,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant)
AND created_at >= @from
AND created_at <= @to
ORDER BY created_at DESC
LIMIT @limit;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "from", from);
AddParameter(command, "to", to);
AddParameter(command, "limit", limit);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByTraceIdAsync(
string tenant,
string traceId,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant) AND trace_id = @trace_id
ORDER BY created_at DESC;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "trace_id", traceId);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByProviderAsync(
string tenant,
string providerId,
int limit,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant) AND LOWER(provider_id) = LOWER(@provider_id)
ORDER BY created_at DESC
LIMIT @limit;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "provider_id", providerId);
AddParameter(command, "limit", limit);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByEventTypeAsync(
string tenant,
string eventType,
int limit,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant) AND LOWER(event_type) = LOWER(@event_type)
ORDER BY created_at DESC
LIMIT @limit;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "event_type", eventType);
AddParameter(command, "limit", limit);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyList<TimelineEvent>> GetRecentAsync(
string tenant,
int limit,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant)
ORDER BY created_at DESC
LIMIT @limit;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "limit", limit);
return await ExecuteQueryAsync(command, cancellationToken).ConfigureAwait(false);
}
public async ValueTask<TimelineEvent?> GetByIdAsync(
string tenant,
string eventId,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT event_id, tenant, provider_id, stream_id, event_type, trace_id,
justification_summary, evidence_hash, payload_hash, created_at, attributes
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant) AND event_id = @event_id;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "event_id", eventId);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
return null;
}
return Map(reader);
}
public async ValueTask<long> CountAsync(string tenant, CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = "SELECT COUNT(*) FROM vex.timeline_events WHERE LOWER(tenant) = LOWER(@tenant);";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt64(result);
}
public async ValueTask<long> CountInRangeAsync(
string tenant,
DateTimeOffset from,
DateTimeOffset to,
CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT COUNT(*)
FROM vex.timeline_events
WHERE LOWER(tenant) = LOWER(@tenant)
AND created_at >= @from
AND created_at <= @to;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "tenant", tenant);
AddParameter(command, "from", from);
AddParameter(command, "to", to);
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt64(result);
}
private static async Task<IReadOnlyList<TimelineEvent>> ExecuteQueryAsync(NpgsqlCommand command, CancellationToken cancellationToken)
{
var results = new List<TimelineEvent>();
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
results.Add(Map(reader));
}
return results;
}
private static TimelineEvent Map(NpgsqlDataReader reader)
{
var eventId = reader.GetString(0);
var tenant = reader.GetString(1);
var providerId = reader.GetString(2);
var streamId = reader.GetString(3);
var eventType = reader.GetString(4);
var traceId = reader.GetString(5);
var justificationSummary = reader.GetString(6);
var evidenceHash = reader.IsDBNull(7) ? null : reader.GetString(7);
var payloadHash = reader.IsDBNull(8) ? null : reader.GetString(8);
var createdAt = reader.GetFieldValue<DateTimeOffset>(9);
var attributesJson = reader.IsDBNull(10) ? null : reader.GetFieldValue<string>(10);
var attributes = DeserializeAttributes(attributesJson);
return new TimelineEvent(
eventId,
tenant,
providerId,
streamId,
eventType,
traceId,
justificationSummary,
createdAt,
evidenceHash,
payloadHash,
attributes);
}
private static string SerializeAttributes(ImmutableDictionary<string, string> attributes)
{
if (attributes.IsEmpty)
{
return "{}";
}
return JsonSerializer.Serialize(attributes);
}
private static ImmutableDictionary<string, string> DeserializeAttributes(string? json)
{
if (string.IsNullOrWhiteSpace(json))
{
return ImmutableDictionary<string, string>.Empty;
}
try
{
using var doc = JsonDocument.Parse(json);
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var property in doc.RootElement.EnumerateObject())
{
if (property.Value.ValueKind == JsonValueKind.String)
{
var value = property.Value.GetString();
if (value is not null)
{
builder[property.Name] = value;
}
}
}
return builder.ToImmutable();
}
catch
{
return ImmutableDictionary<string, string>.Empty;
}
}
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
{
if (_initialized)
{
return;
}
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_initialized)
{
return;
}
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
const string sql = """
CREATE TABLE IF NOT EXISTS vex.timeline_events (
event_id TEXT NOT NULL,
tenant TEXT NOT NULL,
provider_id TEXT NOT NULL,
stream_id TEXT NOT NULL,
event_type TEXT NOT NULL,
trace_id TEXT NOT NULL,
justification_summary TEXT NOT NULL DEFAULT '',
evidence_hash TEXT,
payload_hash TEXT,
created_at TIMESTAMPTZ NOT NULL,
attributes JSONB NOT NULL DEFAULT '{}',
PRIMARY KEY (tenant, event_id)
);
CREATE INDEX IF NOT EXISTS idx_timeline_events_tenant ON vex.timeline_events(tenant);
CREATE INDEX IF NOT EXISTS idx_timeline_events_trace_id ON vex.timeline_events(tenant, trace_id);
CREATE INDEX IF NOT EXISTS idx_timeline_events_provider ON vex.timeline_events(tenant, provider_id);
CREATE INDEX IF NOT EXISTS idx_timeline_events_type ON vex.timeline_events(tenant, event_type);
CREATE INDEX IF NOT EXISTS idx_timeline_events_created_at ON vex.timeline_events(tenant, created_at DESC);
""";
await using var command = CreateCommand(sql, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_initialized = true;
}
finally
{
_initLock.Release();
}
}
}

13
src/Excititor/__Libraries/StellaOps.Excititor.Storage.Postgres/ServiceCollectionExtensions.cs
View File

@@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Excititor.Core.Evidence;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.Storage.Postgres.Repositories;
@@ -39,6 +40,12 @@ public static class ServiceCollectionExtensions
// Register append-only checkpoint store for deterministic persistence (EXCITITOR-ORCH-32/33)
services.AddScoped<IAppendOnlyCheckpointStore, PostgresAppendOnlyCheckpointStore>();
// Register VEX auxiliary stores (SPRINT-3412: PostgreSQL durability)
services.AddScoped<IVexProviderStore, PostgresVexProviderStore>();
services.AddScoped<IVexObservationStore, PostgresVexObservationStore>();
services.AddScoped<IVexAttestationStore, PostgresVexAttestationStore>();
services.AddScoped<IVexTimelineEventStore, PostgresVexTimelineEventStore>();
return services;
}
@@ -65,6 +72,12 @@ public static class ServiceCollectionExtensions
// Register append-only checkpoint store for deterministic persistence (EXCITITOR-ORCH-32/33)
services.AddScoped<IAppendOnlyCheckpointStore, PostgresAppendOnlyCheckpointStore>();
// Register VEX auxiliary stores (SPRINT-3412: PostgreSQL durability)
services.AddScoped<IVexProviderStore, PostgresVexProviderStore>();
services.AddScoped<IVexObservationStore, PostgresVexObservationStore>();
services.AddScoped<IVexAttestationStore, PostgresVexAttestationStore>();
services.AddScoped<IVexTimelineEventStore, PostgresVexTimelineEventStore>();
return services;
}
}