up

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres.Tests/GraphIndexerPostgresFixture.cs

@@ -0,0 +1,26 @@
+using System.Reflection;
+using StellaOps.Infrastructure.Postgres.Testing;
+using Xunit;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres.Tests;
+
+/// <summary>
+/// PostgreSQL integration test fixture for the Graph.Indexer module.
+/// </summary>
+public sealed class GraphIndexerPostgresFixture : PostgresIntegrationFixture, ICollectionFixture<GraphIndexerPostgresFixture>
+{
+    protected override Assembly? GetMigrationAssembly()
+        => typeof(GraphIndexerDataSource).Assembly;
+
+    protected override string GetModuleName() => "GraphIndexer";
+}
+
+/// <summary>
+/// Collection definition for Graph.Indexer PostgreSQL integration tests.
+/// Tests in this collection share a single PostgreSQL container instance.
+/// </summary>
+[CollectionDefinition(Name)]
+public sealed class GraphIndexerPostgresCollection : ICollectionFixture<GraphIndexerPostgresFixture>
+{
+    public const string Name = "GraphIndexerPostgres";
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres.Tests/PostgresIdempotencyStoreTests.cs

@@ -0,0 +1,91 @@
+using FluentAssertions;
+using Microsoft.Extensions.Logging.Abstractions;
+using MicrosoftOptions = Microsoft.Extensions.Options;
+using StellaOps.Graph.Indexer.Storage.Postgres.Repositories;
+using Xunit;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres.Tests;
+
+[Collection(GraphIndexerPostgresCollection.Name)]
+public sealed class PostgresIdempotencyStoreTests : IAsyncLifetime
+{
+    private readonly GraphIndexerPostgresFixture _fixture;
+    private readonly PostgresIdempotencyStore _store;
+
+    public PostgresIdempotencyStoreTests(GraphIndexerPostgresFixture fixture)
+    {
+        _fixture = fixture;
+
+        var options = fixture.Fixture.CreateOptions();
+        options.SchemaName = fixture.SchemaName;
+        var dataSource = new GraphIndexerDataSource(MicrosoftOptions.Options.Create(options), NullLogger<GraphIndexerDataSource>.Instance);
+        _store = new PostgresIdempotencyStore(dataSource, NullLogger<PostgresIdempotencyStore>.Instance);
+    }
+
+    public async Task InitializeAsync()
+    {
+        await _fixture.TruncateAllTablesAsync();
+    }
+
+    public Task DisposeAsync() => Task.CompletedTask;
+
+    [Fact]
+    public async Task HasSeenAsync_ReturnsFalseForNewToken()
+    {
+        // Arrange
+        var sequenceToken = "seq-" + Guid.NewGuid().ToString("N");
+
+        // Act
+        var result = await _store.HasSeenAsync(sequenceToken, CancellationToken.None);
+
+        // Assert
+        result.Should().BeFalse();
+    }
+
+    [Fact]
+    public async Task MarkSeenAsync_ThenHasSeenAsync_ReturnsTrue()
+    {
+        // Arrange
+        var sequenceToken = "seq-" + Guid.NewGuid().ToString("N");
+
+        // Act
+        await _store.MarkSeenAsync(sequenceToken, CancellationToken.None);
+        var result = await _store.HasSeenAsync(sequenceToken, CancellationToken.None);
+
+        // Assert
+        result.Should().BeTrue();
+    }
+
+    [Fact]
+    public async Task MarkSeenAsync_AllowsDifferentTokens()
+    {
+        // Arrange
+        var token1 = "seq-" + Guid.NewGuid().ToString("N");
+        var token2 = "seq-" + Guid.NewGuid().ToString("N");
+
+        // Act
+        await _store.MarkSeenAsync(token1, CancellationToken.None);
+        await _store.MarkSeenAsync(token2, CancellationToken.None);
+        var seen1 = await _store.HasSeenAsync(token1, CancellationToken.None);
+        var seen2 = await _store.HasSeenAsync(token2, CancellationToken.None);
+
+        // Assert
+        seen1.Should().BeTrue();
+        seen2.Should().BeTrue();
+    }
+
+    [Fact]
+    public async Task MarkSeenAsync_IsIdempotent()
+    {
+        // Arrange
+        var sequenceToken = "seq-" + Guid.NewGuid().ToString("N");
+
+        // Act - marking same token twice should not throw
+        await _store.MarkSeenAsync(sequenceToken, CancellationToken.None);
+        await _store.MarkSeenAsync(sequenceToken, CancellationToken.None);
+        var result = await _store.HasSeenAsync(sequenceToken, CancellationToken.None);
+
+        // Assert
+        result.Should().BeTrue();
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres.Tests/StellaOps.Graph.Indexer.Storage.Postgres.Tests.csproj

@@ -0,0 +1,34 @@
+<?xml version="1.0" ?>
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <LangVersion>preview</LangVersion>
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="FluentAssertions" Version="6.12.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="6.0.4">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\StellaOps.Graph.Indexer.Storage.Postgres\StellaOps.Graph.Indexer.Storage.Postgres.csproj" />
+    <ProjectReference Include="..\..\__Libraries\StellaOps.Infrastructure.Postgres.Testing\StellaOps.Infrastructure.Postgres.Testing.csproj" />
+  </ItemGroup>
+
+</Project>

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/GraphIndexerDataSource.cs

@@ -0,0 +1,44 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Npgsql;
+using StellaOps.Infrastructure.Postgres.Connections;
+using StellaOps.Infrastructure.Postgres.Options;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres;
+
+/// <summary>
+/// PostgreSQL data source for Graph.Indexer module.
+/// </summary>
+public sealed class GraphIndexerDataSource : DataSourceBase
+{
+    /// <summary>
+    /// Default schema name for Graph.Indexer tables.
+    /// </summary>
+    public const string DefaultSchemaName = "graph";
+
+    /// <summary>
+    /// Creates a new Graph.Indexer data source.
+    /// </summary>
+    public GraphIndexerDataSource(IOptions<PostgresOptions> options, ILogger<GraphIndexerDataSource> logger)
+        : base(CreateOptions(options.Value), logger)
+    {
+    }
+
+    /// <inheritdoc />
+    protected override string ModuleName => "Graph.Indexer";
+
+    /// <inheritdoc />
+    protected override void ConfigureDataSourceBuilder(NpgsqlDataSourceBuilder builder)
+    {
+        base.ConfigureDataSourceBuilder(builder);
+    }
+
+    private static PostgresOptions CreateOptions(PostgresOptions baseOptions)
+    {
+        if (string.IsNullOrWhiteSpace(baseOptions.SchemaName))
+        {
+            baseOptions.SchemaName = DefaultSchemaName;
+        }
+        return baseOptions;
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/Repositories/PostgresGraphAnalyticsWriter.cs

@@ -0,0 +1,181 @@
+using System.Collections.Immutable;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using StellaOps.Graph.Indexer.Analytics;
+using StellaOps.Infrastructure.Postgres.Repositories;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres.Repositories;
+
+/// <summary>
+/// PostgreSQL implementation of <see cref="IGraphAnalyticsWriter"/>.
+/// </summary>
+public sealed class PostgresGraphAnalyticsWriter : RepositoryBase<GraphIndexerDataSource>, IGraphAnalyticsWriter
+{
+    private bool _tableInitialized;
+
+    public PostgresGraphAnalyticsWriter(GraphIndexerDataSource dataSource, ILogger<PostgresGraphAnalyticsWriter> logger)
+        : base(dataSource, logger)
+    {
+    }
+
+    public async Task PersistClusterAssignmentsAsync(
+        GraphAnalyticsSnapshot snapshot,
+        ImmutableArray<ClusterAssignment> assignments,
+        CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(snapshot);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
+
+        try
+        {
+            // Delete existing assignments for this snapshot
+            const string deleteSql = @"
+                DELETE FROM graph.cluster_assignments
+                WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
+
+            await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
+            {
+                AddParameter(deleteCommand, "@tenant", snapshot.Tenant ?? string.Empty);
+                AddParameter(deleteCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
+                await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            }
+
+            // Insert new assignments
+            const string insertSql = @"
+                INSERT INTO graph.cluster_assignments (tenant, snapshot_id, node_id, cluster_id, kind, computed_at)
+                VALUES (@tenant, @snapshot_id, @node_id, @cluster_id, @kind, @computed_at)";
+
+            var computedAt = snapshot.GeneratedAt;
+
+            foreach (var assignment in assignments)
+            {
+                await using var insertCommand = CreateCommand(insertSql, connection, transaction);
+                AddParameter(insertCommand, "@tenant", snapshot.Tenant ?? string.Empty);
+                AddParameter(insertCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
+                AddParameter(insertCommand, "@node_id", assignment.NodeId ?? string.Empty);
+                AddParameter(insertCommand, "@cluster_id", assignment.ClusterId ?? string.Empty);
+                AddParameter(insertCommand, "@kind", assignment.Kind ?? string.Empty);
+                AddParameter(insertCommand, "@computed_at", computedAt);
+
+                await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            }
+
+            await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch
+        {
+            await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
+            throw;
+        }
+    }
+
+    public async Task PersistCentralityAsync(
+        GraphAnalyticsSnapshot snapshot,
+        ImmutableArray<CentralityScore> scores,
+        CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(snapshot);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
+
+        try
+        {
+            // Delete existing scores for this snapshot
+            const string deleteSql = @"
+                DELETE FROM graph.centrality_scores
+                WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
+
+            await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
+            {
+                AddParameter(deleteCommand, "@tenant", snapshot.Tenant ?? string.Empty);
+                AddParameter(deleteCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
+                await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            }
+
+            // Insert new scores
+            const string insertSql = @"
+                INSERT INTO graph.centrality_scores (tenant, snapshot_id, node_id, degree, betweenness, kind, computed_at)
+                VALUES (@tenant, @snapshot_id, @node_id, @degree, @betweenness, @kind, @computed_at)";
+
+            var computedAt = snapshot.GeneratedAt;
+
+            foreach (var score in scores)
+            {
+                await using var insertCommand = CreateCommand(insertSql, connection, transaction);
+                AddParameter(insertCommand, "@tenant", snapshot.Tenant ?? string.Empty);
+                AddParameter(insertCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
+                AddParameter(insertCommand, "@node_id", score.NodeId ?? string.Empty);
+                AddParameter(insertCommand, "@degree", score.Degree);
+                AddParameter(insertCommand, "@betweenness", score.Betweenness);
+                AddParameter(insertCommand, "@kind", score.Kind ?? string.Empty);
+                AddParameter(insertCommand, "@computed_at", computedAt);
+
+                await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            }
+
+            await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch
+        {
+            await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
+            throw;
+        }
+    }
+
+    private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
+    {
+        return new NpgsqlCommand(sql, connection, transaction);
+    }
+
+    private async Task EnsureTableAsync(CancellationToken cancellationToken)
+    {
+        if (_tableInitialized)
+        {
+            return;
+        }
+
+        const string ddl = @"
+            CREATE SCHEMA IF NOT EXISTS graph;
+
+            CREATE TABLE IF NOT EXISTS graph.cluster_assignments (
+                tenant TEXT NOT NULL,
+                snapshot_id TEXT NOT NULL,
+                node_id TEXT NOT NULL,
+                cluster_id TEXT NOT NULL,
+                kind TEXT NOT NULL,
+                computed_at TIMESTAMPTZ NOT NULL,
+                PRIMARY KEY (tenant, snapshot_id, node_id)
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_cluster_assignments_cluster ON graph.cluster_assignments (tenant, cluster_id);
+            CREATE INDEX IF NOT EXISTS idx_cluster_assignments_computed_at ON graph.cluster_assignments (computed_at);
+
+            CREATE TABLE IF NOT EXISTS graph.centrality_scores (
+                tenant TEXT NOT NULL,
+                snapshot_id TEXT NOT NULL,
+                node_id TEXT NOT NULL,
+                degree DOUBLE PRECISION NOT NULL,
+                betweenness DOUBLE PRECISION NOT NULL,
+                kind TEXT NOT NULL,
+                computed_at TIMESTAMPTZ NOT NULL,
+                PRIMARY KEY (tenant, snapshot_id, node_id)
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_centrality_scores_degree ON graph.centrality_scores (tenant, degree DESC);
+            CREATE INDEX IF NOT EXISTS idx_centrality_scores_betweenness ON graph.centrality_scores (tenant, betweenness DESC);
+            CREATE INDEX IF NOT EXISTS idx_centrality_scores_computed_at ON graph.centrality_scores (computed_at);";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(ddl, connection);
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+
+        _tableInitialized = true;
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/Repositories/PostgresGraphDocumentWriter.cs

@@ -0,0 +1,174 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using StellaOps.Graph.Indexer.Ingestion.Sbom;
+using StellaOps.Infrastructure.Postgres.Repositories;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres.Repositories;
+
+/// <summary>
+/// PostgreSQL implementation of <see cref="IGraphDocumentWriter"/>.
+/// </summary>
+public sealed class PostgresGraphDocumentWriter : RepositoryBase<GraphIndexerDataSource>, IGraphDocumentWriter
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        WriteIndented = false
+    };
+
+    private bool _tableInitialized;
+
+    public PostgresGraphDocumentWriter(GraphIndexerDataSource dataSource, ILogger<PostgresGraphDocumentWriter> logger)
+        : base(dataSource, logger)
+    {
+    }
+
+    public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(batch);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
+
+        try
+        {
+            var batchId = Guid.NewGuid().ToString("N");
+            var writtenAt = DateTimeOffset.UtcNow;
+
+            // Insert nodes
+            foreach (var node in batch.Nodes)
+            {
+                var nodeId = ExtractId(node);
+                var nodeJson = node.ToJsonString();
+
+                const string nodeSql = @"
+                    INSERT INTO graph.graph_nodes (id, batch_id, document_json, written_at)
+                    VALUES (@id, @batch_id, @document_json, @written_at)
+                    ON CONFLICT (id) DO UPDATE SET
+                        batch_id = EXCLUDED.batch_id,
+                        document_json = EXCLUDED.document_json,
+                        written_at = EXCLUDED.written_at";
+
+                await using var nodeCommand = CreateCommand(nodeSql, connection, transaction);
+                AddParameter(nodeCommand, "@id", nodeId);
+                AddParameter(nodeCommand, "@batch_id", batchId);
+                AddJsonbParameter(nodeCommand, "@document_json", nodeJson);
+                AddParameter(nodeCommand, "@written_at", writtenAt);
+
+                await nodeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            }
+
+            // Insert edges
+            foreach (var edge in batch.Edges)
+            {
+                var edgeId = ExtractEdgeId(edge);
+                var edgeJson = edge.ToJsonString();
+
+                const string edgeSql = @"
+                    INSERT INTO graph.graph_edges (id, batch_id, source_id, target_id, document_json, written_at)
+                    VALUES (@id, @batch_id, @source_id, @target_id, @document_json, @written_at)
+                    ON CONFLICT (id) DO UPDATE SET
+                        batch_id = EXCLUDED.batch_id,
+                        source_id = EXCLUDED.source_id,
+                        target_id = EXCLUDED.target_id,
+                        document_json = EXCLUDED.document_json,
+                        written_at = EXCLUDED.written_at";
+
+                await using var edgeCommand = CreateCommand(edgeSql, connection, transaction);
+                AddParameter(edgeCommand, "@id", edgeId);
+                AddParameter(edgeCommand, "@batch_id", batchId);
+                AddParameter(edgeCommand, "@source_id", ExtractString(edge, "source") ?? string.Empty);
+                AddParameter(edgeCommand, "@target_id", ExtractString(edge, "target") ?? string.Empty);
+                AddJsonbParameter(edgeCommand, "@document_json", edgeJson);
+                AddParameter(edgeCommand, "@written_at", writtenAt);
+
+                await edgeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+            }
+
+            await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch
+        {
+            await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
+            throw;
+        }
+    }
+
+    private static string ExtractId(JsonObject node)
+    {
+        return ExtractString(node, "id") ?? ExtractString(node, "@id") ?? Guid.NewGuid().ToString("N");
+    }
+
+    private static string ExtractEdgeId(JsonObject edge)
+    {
+        var id = ExtractString(edge, "id") ?? ExtractString(edge, "@id");
+        if (!string.IsNullOrWhiteSpace(id))
+        {
+            return id;
+        }
+
+        var source = ExtractString(edge, "source") ?? string.Empty;
+        var target = ExtractString(edge, "target") ?? string.Empty;
+        var type = ExtractString(edge, "type") ?? ExtractString(edge, "relationship") ?? "relates_to";
+        return $"{source}|{target}|{type}";
+    }
+
+    private static string? ExtractString(JsonObject obj, string key)
+    {
+        if (obj.TryGetPropertyValue(key, out var value) && value is JsonValue jv)
+        {
+            return jv.GetValue<string>();
+        }
+        return null;
+    }
+
+    private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
+    {
+        return new NpgsqlCommand(sql, connection, transaction);
+    }
+
+    private async Task EnsureTableAsync(CancellationToken cancellationToken)
+    {
+        if (_tableInitialized)
+        {
+            return;
+        }
+
+        const string ddl = @"
+            CREATE SCHEMA IF NOT EXISTS graph;
+
+            CREATE TABLE IF NOT EXISTS graph.graph_nodes (
+                id TEXT PRIMARY KEY,
+                batch_id TEXT NOT NULL,
+                document_json JSONB NOT NULL,
+                written_at TIMESTAMPTZ NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_graph_nodes_batch_id ON graph.graph_nodes (batch_id);
+            CREATE INDEX IF NOT EXISTS idx_graph_nodes_written_at ON graph.graph_nodes (written_at);
+
+            CREATE TABLE IF NOT EXISTS graph.graph_edges (
+                id TEXT PRIMARY KEY,
+                batch_id TEXT NOT NULL,
+                source_id TEXT NOT NULL,
+                target_id TEXT NOT NULL,
+                document_json JSONB NOT NULL,
+                written_at TIMESTAMPTZ NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_graph_edges_batch_id ON graph.graph_edges (batch_id);
+            CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id ON graph.graph_edges (source_id);
+            CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id ON graph.graph_edges (target_id);
+            CREATE INDEX IF NOT EXISTS idx_graph_edges_written_at ON graph.graph_edges (written_at);";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(ddl, connection);
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+
+        _tableInitialized = true;
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/Repositories/PostgresGraphSnapshotProvider.cs

@@ -0,0 +1,157 @@
+using System.Collections.Immutable;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using StellaOps.Graph.Indexer.Analytics;
+using StellaOps.Infrastructure.Postgres.Repositories;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres.Repositories;
+
+/// <summary>
+/// PostgreSQL implementation of <see cref="IGraphSnapshotProvider"/>.
+/// </summary>
+public sealed class PostgresGraphSnapshotProvider : RepositoryBase<GraphIndexerDataSource>, IGraphSnapshotProvider
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        WriteIndented = false
+    };
+
+    private bool _tableInitialized;
+
+    public PostgresGraphSnapshotProvider(GraphIndexerDataSource dataSource, ILogger<PostgresGraphSnapshotProvider> logger)
+        : base(dataSource, logger)
+    {
+    }
+
+    /// <summary>
+    /// Enqueues a snapshot for processing.
+    /// </summary>
+    public async Task EnqueueAsync(GraphAnalyticsSnapshot snapshot, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(snapshot);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        const string sql = @"
+            INSERT INTO graph.pending_snapshots (tenant, snapshot_id, generated_at, nodes_json, edges_json, queued_at)
+            VALUES (@tenant, @snapshot_id, @generated_at, @nodes_json, @edges_json, @queued_at)
+            ON CONFLICT (tenant, snapshot_id) DO UPDATE SET
+                generated_at = EXCLUDED.generated_at,
+                nodes_json = EXCLUDED.nodes_json,
+                edges_json = EXCLUDED.edges_json,
+                queued_at = EXCLUDED.queued_at";
+
+        var nodesJson = JsonSerializer.Serialize(snapshot.Nodes.Select(n => n.ToJsonString()), JsonOptions);
+        var edgesJson = JsonSerializer.Serialize(snapshot.Edges.Select(e => e.ToJsonString()), JsonOptions);
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(sql, connection);
+        AddParameter(command, "@tenant", snapshot.Tenant ?? string.Empty);
+        AddParameter(command, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
+        AddParameter(command, "@generated_at", snapshot.GeneratedAt);
+        AddJsonbParameter(command, "@nodes_json", nodesJson);
+        AddJsonbParameter(command, "@edges_json", edgesJson);
+        AddParameter(command, "@queued_at", DateTimeOffset.UtcNow);
+
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    public async Task<IReadOnlyList<GraphAnalyticsSnapshot>> GetPendingSnapshotsAsync(CancellationToken cancellationToken)
+    {
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        const string sql = @"
+            SELECT tenant, snapshot_id, generated_at, nodes_json, edges_json
+            FROM graph.pending_snapshots
+            ORDER BY queued_at ASC
+            LIMIT 100";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(sql, connection);
+        await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
+
+        var results = new List<GraphAnalyticsSnapshot>();
+        while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+        {
+            results.Add(MapSnapshot(reader));
+        }
+
+        return results.ToImmutableArray();
+    }
+
+    public async Task MarkProcessedAsync(string tenant, string snapshotId, CancellationToken cancellationToken)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(snapshotId);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        const string sql = @"
+            DELETE FROM graph.pending_snapshots
+            WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(sql, connection);
+        AddParameter(command, "@tenant", tenant ?? string.Empty);
+        AddParameter(command, "@snapshot_id", snapshotId.Trim());
+
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    private static GraphAnalyticsSnapshot MapSnapshot(NpgsqlDataReader reader)
+    {
+        var tenant = reader.GetString(0);
+        var snapshotId = reader.GetString(1);
+        var generatedAt = reader.GetFieldValue<DateTimeOffset>(2);
+        var nodesJson = reader.GetString(3);
+        var edgesJson = reader.GetString(4);
+
+        var nodeStrings = JsonSerializer.Deserialize<List<string>>(nodesJson, JsonOptions) ?? new List<string>();
+        var edgeStrings = JsonSerializer.Deserialize<List<string>>(edgesJson, JsonOptions) ?? new List<string>();
+
+        var nodes = nodeStrings
+            .Select(s => JsonNode.Parse(s) as JsonObject)
+            .Where(n => n is not null)
+            .Cast<JsonObject>()
+            .ToImmutableArray();
+
+        var edges = edgeStrings
+            .Select(s => JsonNode.Parse(s) as JsonObject)
+            .Where(e => e is not null)
+            .Cast<JsonObject>()
+            .ToImmutableArray();
+
+        return new GraphAnalyticsSnapshot(tenant, snapshotId, generatedAt, nodes, edges);
+    }
+
+    private async Task EnsureTableAsync(CancellationToken cancellationToken)
+    {
+        if (_tableInitialized)
+        {
+            return;
+        }
+
+        const string ddl = @"
+            CREATE SCHEMA IF NOT EXISTS graph;
+
+            CREATE TABLE IF NOT EXISTS graph.pending_snapshots (
+                tenant TEXT NOT NULL,
+                snapshot_id TEXT NOT NULL,
+                generated_at TIMESTAMPTZ NOT NULL,
+                nodes_json JSONB NOT NULL,
+                edges_json JSONB NOT NULL,
+                queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                PRIMARY KEY (tenant, snapshot_id)
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_pending_snapshots_queued_at ON graph.pending_snapshots (queued_at);";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(ddl, connection);
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+
+        _tableInitialized = true;
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/Repositories/PostgresIdempotencyStore.cs

@@ -0,0 +1,78 @@
+using Microsoft.Extensions.Logging;
+using StellaOps.Graph.Indexer.Incremental;
+using StellaOps.Infrastructure.Postgres.Repositories;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres.Repositories;
+
+/// <summary>
+/// PostgreSQL implementation of <see cref="IIdempotencyStore"/>.
+/// </summary>
+public sealed class PostgresIdempotencyStore : RepositoryBase<GraphIndexerDataSource>, IIdempotencyStore
+{
+    private bool _tableInitialized;
+
+    public PostgresIdempotencyStore(GraphIndexerDataSource dataSource, ILogger<PostgresIdempotencyStore> logger)
+        : base(dataSource, logger)
+    {
+    }
+
+    public async Task<bool> HasSeenAsync(string sequenceToken, CancellationToken cancellationToken)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(sequenceToken);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        const string sql = @"
+            SELECT EXISTS(SELECT 1 FROM graph.idempotency_tokens WHERE sequence_token = @sequence_token)";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(sql, connection);
+        AddParameter(command, "@sequence_token", sequenceToken.Trim());
+
+        var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+        return result is bool seen && seen;
+    }
+
+    public async Task MarkSeenAsync(string sequenceToken, CancellationToken cancellationToken)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(sequenceToken);
+
+        await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
+
+        const string sql = @"
+            INSERT INTO graph.idempotency_tokens (sequence_token, seen_at)
+            VALUES (@sequence_token, @seen_at)
+            ON CONFLICT (sequence_token) DO NOTHING";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(sql, connection);
+        AddParameter(command, "@sequence_token", sequenceToken.Trim());
+        AddParameter(command, "@seen_at", DateTimeOffset.UtcNow);
+
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    private async Task EnsureTableAsync(CancellationToken cancellationToken)
+    {
+        if (_tableInitialized)
+        {
+            return;
+        }
+
+        const string ddl = @"
+            CREATE SCHEMA IF NOT EXISTS graph;
+
+            CREATE TABLE IF NOT EXISTS graph.idempotency_tokens (
+                sequence_token TEXT PRIMARY KEY,
+                seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_seen_at ON graph.idempotency_tokens (seen_at);";
+
+        await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
+        await using var command = CreateCommand(ddl, connection);
+        await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+
+        _tableInitialized = true;
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/ServiceCollectionExtensions.cs

@@ -0,0 +1,61 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using StellaOps.Graph.Indexer.Analytics;
+using StellaOps.Graph.Indexer.Incremental;
+using StellaOps.Graph.Indexer.Ingestion.Sbom;
+using StellaOps.Graph.Indexer.Storage.Postgres.Repositories;
+using StellaOps.Infrastructure.Postgres.Options;
+
+namespace StellaOps.Graph.Indexer.Storage.Postgres;
+
+/// <summary>
+/// Extension methods for configuring Graph.Indexer PostgreSQL storage services.
+/// </summary>
+public static class ServiceCollectionExtensions
+{
+    /// <summary>
+    /// Adds Graph.Indexer PostgreSQL storage services.
+    /// </summary>
+    /// <param name="services">Service collection.</param>
+    /// <param name="configuration">Configuration root.</param>
+    /// <param name="sectionName">Configuration section name for PostgreSQL options.</param>
+    /// <returns>Service collection for chaining.</returns>
+    public static IServiceCollection AddGraphIndexerPostgresStorage(
+        this IServiceCollection services,
+        IConfiguration configuration,
+        string sectionName = "Postgres:Graph")
+    {
+        services.Configure<PostgresOptions>(configuration.GetSection(sectionName));
+        services.AddSingleton<GraphIndexerDataSource>();
+
+        // Register repositories
+        services.AddSingleton<IIdempotencyStore, PostgresIdempotencyStore>();
+        services.AddSingleton<IGraphSnapshotProvider, PostgresGraphSnapshotProvider>();
+        services.AddSingleton<IGraphAnalyticsWriter, PostgresGraphAnalyticsWriter>();
+        services.AddSingleton<IGraphDocumentWriter, PostgresGraphDocumentWriter>();
+
+        return services;
+    }
+
+    /// <summary>
+    /// Adds Graph.Indexer PostgreSQL storage services with explicit options.
+    /// </summary>
+    /// <param name="services">Service collection.</param>
+    /// <param name="configureOptions">Options configuration action.</param>
+    /// <returns>Service collection for chaining.</returns>
+    public static IServiceCollection AddGraphIndexerPostgresStorage(
+        this IServiceCollection services,
+        Action<PostgresOptions> configureOptions)
+    {
+        services.Configure(configureOptions);
+        services.AddSingleton<GraphIndexerDataSource>();
+
+        // Register repositories
+        services.AddSingleton<IIdempotencyStore, PostgresIdempotencyStore>();
+        services.AddSingleton<IGraphSnapshotProvider, PostgresGraphSnapshotProvider>();
+        services.AddSingleton<IGraphAnalyticsWriter, PostgresGraphAnalyticsWriter>();
+        services.AddSingleton<IGraphDocumentWriter, PostgresGraphDocumentWriter>();
+
+        return services;
+    }
+}

src/Graph/StellaOps.Graph.Indexer.Storage.Postgres/StellaOps.Graph.Indexer.Storage.Postgres.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <RootNamespace>StellaOps.Graph.Indexer.Storage.Postgres</RootNamespace>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="../StellaOps.Graph.Indexer/StellaOps.Graph.Indexer.csproj" />
+    <ProjectReference Include="../../__Libraries/StellaOps.Infrastructure.Postgres/StellaOps.Infrastructure.Postgres.csproj" />
+  </ItemGroup>
+</Project>

src/Graph/StellaOps.Graph.Indexer/Documents/GraphSnapshotBuilder.cs

@@ -289,8 +289,31 @@ public sealed class GraphSnapshotBuilder
         out string sourceNodeId,
         out string targetNodeId)
     {
-        var kind = edge["kind"]!.GetValue<string>();
-        var canonicalKey = edge["canonical_key"]!.AsObject();
+        // Handle simple edge format with direct source/target properties
+        if (!edge.TryGetPropertyValue("kind", out var kindNode) || kindNode is null)
+        {
+            if (edge.TryGetPropertyValue("source", out var simpleSource) && simpleSource is not null &&
+                edge.TryGetPropertyValue("target", out var simpleTarget) && simpleTarget is not null)
+            {
+                sourceNodeId = simpleSource.GetValue<string>();
+                targetNodeId = simpleTarget.GetValue<string>();
+                return nodesById.ContainsKey(sourceNodeId) && nodesById.ContainsKey(targetNodeId);
+            }
+
+            sourceNodeId = string.Empty;
+            targetNodeId = string.Empty;
+            return false;
+        }
+
+        var kind = kindNode.GetValue<string>();
+        if (!edge.TryGetPropertyValue("canonical_key", out var canonicalKeyNode) || canonicalKeyNode is null)
+        {
+            sourceNodeId = string.Empty;
+            targetNodeId = string.Empty;
+            return false;
+        }
+
+        var canonicalKey = canonicalKeyNode.AsObject();
 
         string? source = null;
         string? target = null;

src/Graph/__Tests/StellaOps.Graph.Indexer.Tests/GraphAnalyticsEngineTests.cs

@@ -14,8 +14,8 @@ public sealed class GraphAnalyticsEngineTests
         var first = engine.Compute(snapshot);
         var second = engine.Compute(snapshot);
 
-        Assert.Equal(first.Clusters, second.Clusters);
-        Assert.Equal(first.CentralityScores, second.CentralityScores);
+        Assert.Equal(first.Clusters.ToArray(), second.Clusters.ToArray());
+        Assert.Equal(first.CentralityScores.ToArray(), second.CentralityScores.ToArray());
 
         var mainCluster = first.Clusters.First(c => c.NodeId == snapshot.Nodes[0]["id"]!.GetValue<string>()).ClusterId;
         Assert.All(first.Clusters.Where(c => c.NodeId != snapshot.Nodes[^1]["id"]!.GetValue<string>()), c => Assert.Equal(mainCluster, c.ClusterId));