stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

wip: doctor/cli/docs/api to vector db consolidation; api hardening for descriptions, tenant, and scopes; migrations and conversions of all DALs to EF v10

Browse Source
This commit is contained in:
master
2026-02-23 15:30:50 +02:00
parent bd8fee6ed8
commit e746577380
1424 changed files with 81225 additions and 25251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Graph/__Libraries/StellaOps.Graph.Core/ICveObservationNodeRepository.cs
View File

@@ -23,6 +23,7 @@ public interface ICveObservationNodeRepository
/// Gets an observation node by ID.
/// </summary>
Task<CveObservationNode?> GetByIdAsync(
string tenantId,
string nodeId,
CancellationToken ct = default);
@@ -83,6 +84,7 @@ public interface ICveObservationNodeRepository
/// Deletes an observation node.
/// </summary>
Task<bool> DeleteAsync(
string tenantId,
string nodeId,
CancellationToken ct = default);

10
src/Graph/__Libraries/StellaOps.Graph.Core/PostgresCveObservationNodeRepository.cs
View File

@@ -94,15 +94,18 @@ public sealed class PostgresCveObservationNodeRepository : ICveObservationNodeRe
/// <inheritdoc />
public async Task<CveObservationNode?> GetByIdAsync(
string tenantId,
string nodeId,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes WHERE node_id = @node_id
SELECT * FROM cve_observation_nodes
WHERE tenant_id = @tenant_id AND node_id = @node_id
""";
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
cmd.Parameters.AddWithValue("tenant_id", tenantId);
cmd.Parameters.AddWithValue("node_id", nodeId);
await using var reader = await cmd.ExecuteReaderAsync(ct);
@@ -221,15 +224,18 @@ public sealed class PostgresCveObservationNodeRepository : ICveObservationNodeRe
/// <inheritdoc />
public async Task<bool> DeleteAsync(
string tenantId,
string nodeId,
CancellationToken ct = default)
{
const string sql = """
DELETE FROM cve_observation_nodes WHERE node_id = @node_id
DELETE FROM cve_observation_nodes
WHERE tenant_id = @tenant_id AND node_id = @node_id
""";
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
cmd.Parameters.AddWithValue("tenant_id", tenantId);
cmd.Parameters.AddWithValue("node_id", nodeId);
var affected = await cmd.ExecuteNonQueryAsync(ct);

37
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/CompiledModels/GraphIndexerDbContextModel.cs Normal file
View File

@@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
#pragma warning disable 219, 612, 618
#nullable disable
namespace StellaOps.Graph.Indexer.Persistence.EfCore.CompiledModels;
/// <summary>
/// Compiled model stub for GraphIndexerDbContext.
/// This is a placeholder that delegates to runtime model building.
/// Replace with output from <c>dotnet ef dbcontext optimize</c> when a provisioned DB is available.
/// </summary>
[DbContext(typeof(Context.GraphIndexerDbContext))]
public partial class GraphIndexerDbContextModel : RuntimeModel
{
private static GraphIndexerDbContextModel _instance;
public static IModel Instance
{
get
{
if (_instance == null)
{
_instance = new GraphIndexerDbContextModel();
_instance.Initialize();
_instance.Customize();
}
return _instance;
}
}
partial void Initialize();
partial void Customize();
}

20
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/CompiledModels/GraphIndexerDbContextModelBuilder.cs Normal file
View File

@@ -0,0 +1,20 @@
using Microsoft.EntityFrameworkCore.Infrastructure;
#pragma warning disable 219, 612, 618
#nullable disable
namespace StellaOps.Graph.Indexer.Persistence.EfCore.CompiledModels;
/// <summary>
/// Compiled model builder stub for GraphIndexerDbContext.
/// Replace with output from <c>dotnet ef dbcontext optimize</c> when a provisioned DB is available.
/// </summary>
public partial class GraphIndexerDbContextModel
{
partial void Initialize()
{
// Stub: when a real compiled model is generated, entity types will be registered here.
// The runtime factory will fall back to reflection-based model building for all schemas
// until this stub is replaced with a full compiled model.
}
}

12
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Context/GraphIndexerDbContext.Partial.cs Normal file
View File

@@ -0,0 +1,12 @@
using Microsoft.EntityFrameworkCore;
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Context;
public partial class GraphIndexerDbContext
{
partial void OnModelCreatingPartial(ModelBuilder modelBuilder)
{
// No navigation property overlays needed for Graph Indexer;
// all tables are standalone with no foreign key relationships.
}
}

139
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Context/GraphIndexerDbContext.cs
View File

@@ -1,21 +1,150 @@
using Microsoft.EntityFrameworkCore;
using StellaOps.Graph.Indexer.Persistence.EfCore.Models;
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Context;
/// <summary>
/// EF Core DbContext for Graph Indexer module.
/// This is a stub that will be scaffolded from the PostgreSQL database.
/// Maps to the graph PostgreSQL schema: graph_nodes, graph_edges, pending_snapshots,
/// cluster_assignments, centrality_scores, and idempotency_tokens tables.
/// </summary>
public class GraphIndexerDbContext : DbContext
public partial class GraphIndexerDbContext : DbContext
{
public GraphIndexerDbContext(DbContextOptions<GraphIndexerDbContext> options)
private readonly string _schemaName;
public GraphIndexerDbContext(DbContextOptions<GraphIndexerDbContext> options, string? schemaName = null)
: base(options)
{
_schemaName = string.IsNullOrWhiteSpace(schemaName)
? "graph"
: schemaName.Trim();
}
public virtual DbSet<GraphNode> GraphNodes { get; set; }
public virtual DbSet<GraphEdge> GraphEdges { get; set; }
public virtual DbSet<PendingSnapshot> PendingSnapshots { get; set; }
public virtual DbSet<ClusterAssignmentEntity> ClusterAssignments { get; set; }
public virtual DbSet<CentralityScoreEntity> CentralityScores { get; set; }
public virtual DbSet<IdempotencyToken> IdempotencyTokens { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.HasDefaultSchema("graph");
base.OnModelCreating(modelBuilder);
var schemaName = _schemaName;
// -- graph_nodes ----------------------------------------------------------
modelBuilder.Entity<GraphNode>(entity =>
{
entity.HasKey(e => e.Id).HasName("graph_nodes_pkey");
entity.ToTable("graph_nodes", schemaName);
entity.HasIndex(e => e.BatchId, "idx_graph_nodes_batch_id");
entity.HasIndex(e => e.WrittenAt, "idx_graph_nodes_written_at");
entity.Property(e => e.Id).HasColumnName("id");
entity.Property(e => e.BatchId).HasColumnName("batch_id");
entity.Property(e => e.DocumentJson)
.HasColumnType("jsonb")
.HasColumnName("document_json");
entity.Property(e => e.WrittenAt).HasColumnName("written_at");
});
// -- graph_edges ----------------------------------------------------------
modelBuilder.Entity<GraphEdge>(entity =>
{
entity.HasKey(e => e.Id).HasName("graph_edges_pkey");
entity.ToTable("graph_edges", schemaName);
entity.HasIndex(e => e.BatchId, "idx_graph_edges_batch_id");
entity.HasIndex(e => e.SourceId, "idx_graph_edges_source_id");
entity.HasIndex(e => e.TargetId, "idx_graph_edges_target_id");
entity.HasIndex(e => e.WrittenAt, "idx_graph_edges_written_at");
entity.Property(e => e.Id).HasColumnName("id");
entity.Property(e => e.BatchId).HasColumnName("batch_id");
entity.Property(e => e.SourceId).HasColumnName("source_id");
entity.Property(e => e.TargetId).HasColumnName("target_id");
entity.Property(e => e.DocumentJson)
.HasColumnType("jsonb")
.HasColumnName("document_json");
entity.Property(e => e.WrittenAt).HasColumnName("written_at");
});
// -- pending_snapshots ----------------------------------------------------
modelBuilder.Entity<PendingSnapshot>(entity =>
{
entity.HasKey(e => new { e.Tenant, e.SnapshotId }).HasName("pending_snapshots_pkey");
entity.ToTable("pending_snapshots", schemaName);
entity.HasIndex(e => e.QueuedAt, "idx_pending_snapshots_queued_at");
entity.Property(e => e.Tenant).HasColumnName("tenant");
entity.Property(e => e.SnapshotId).HasColumnName("snapshot_id");
entity.Property(e => e.GeneratedAt).HasColumnName("generated_at");
entity.Property(e => e.NodesJson)
.HasColumnType("jsonb")
.HasColumnName("nodes_json");
entity.Property(e => e.EdgesJson)
.HasColumnType("jsonb")
.HasColumnName("edges_json");
entity.Property(e => e.QueuedAt)
.HasDefaultValueSql("now()")
.HasColumnName("queued_at");
});
// -- cluster_assignments --------------------------------------------------
modelBuilder.Entity<ClusterAssignmentEntity>(entity =>
{
entity.HasKey(e => new { e.Tenant, e.SnapshotId, e.NodeId }).HasName("cluster_assignments_pkey");
entity.ToTable("cluster_assignments", schemaName);
entity.HasIndex(e => new { e.Tenant, e.ClusterId }, "idx_cluster_assignments_cluster");
entity.HasIndex(e => e.ComputedAt, "idx_cluster_assignments_computed_at");
entity.Property(e => e.Tenant).HasColumnName("tenant");
entity.Property(e => e.SnapshotId).HasColumnName("snapshot_id");
entity.Property(e => e.NodeId).HasColumnName("node_id");
entity.Property(e => e.ClusterId).HasColumnName("cluster_id");
entity.Property(e => e.Kind).HasColumnName("kind");
entity.Property(e => e.ComputedAt).HasColumnName("computed_at");
});
// -- centrality_scores ----------------------------------------------------
modelBuilder.Entity<CentralityScoreEntity>(entity =>
{
entity.HasKey(e => new { e.Tenant, e.SnapshotId, e.NodeId }).HasName("centrality_scores_pkey");
entity.ToTable("centrality_scores", schemaName);
entity.HasIndex(e => new { e.Tenant, e.Degree }, "idx_centrality_scores_degree")
.IsDescending(false, true);
entity.HasIndex(e => new { e.Tenant, e.Betweenness }, "idx_centrality_scores_betweenness")
.IsDescending(false, true);
entity.HasIndex(e => e.ComputedAt, "idx_centrality_scores_computed_at");
entity.Property(e => e.Tenant).HasColumnName("tenant");
entity.Property(e => e.SnapshotId).HasColumnName("snapshot_id");
entity.Property(e => e.NodeId).HasColumnName("node_id");
entity.Property(e => e.Degree).HasColumnName("degree");
entity.Property(e => e.Betweenness).HasColumnName("betweenness");
entity.Property(e => e.Kind).HasColumnName("kind");
entity.Property(e => e.ComputedAt).HasColumnName("computed_at");
});
// -- idempotency_tokens ---------------------------------------------------
modelBuilder.Entity<IdempotencyToken>(entity =>
{
entity.HasKey(e => e.SequenceToken).HasName("idempotency_tokens_pkey");
entity.ToTable("idempotency_tokens", schemaName);
entity.HasIndex(e => e.SeenAt, "idx_idempotency_tokens_seen_at");
entity.Property(e => e.SequenceToken).HasColumnName("sequence_token");
entity.Property(e => e.SeenAt)
.HasDefaultValueSql("now()")
.HasColumnName("seen_at");
});
OnModelCreatingPartial(modelBuilder);
}
partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
}

31
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Context/GraphIndexerDesignTimeDbContextFactory.cs Normal file
View File

@@ -0,0 +1,31 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Context;
/// <summary>
/// Design-time factory for <c>dotnet ef</c> CLI tooling (scaffold, optimize).
/// </summary>
public sealed class GraphIndexerDesignTimeDbContextFactory : IDesignTimeDbContextFactory<GraphIndexerDbContext>
{
private const string DefaultConnectionString =
"Host=localhost;Port=55433;Database=postgres;Username=postgres;Password=postgres;Search Path=graph,public";
private const string ConnectionStringEnvironmentVariable = "STELLAOPS_GRAPH_EF_CONNECTION";
public GraphIndexerDbContext CreateDbContext(string[] args)
{
var connectionString = ResolveConnectionString();
var options = new DbContextOptionsBuilder<GraphIndexerDbContext>()
.UseNpgsql(connectionString)
.Options;
return new GraphIndexerDbContext(options);
}
private static string ResolveConnectionString()
{
var fromEnvironment = Environment.GetEnvironmentVariable(ConnectionStringEnvironmentVariable);
return string.IsNullOrWhiteSpace(fromEnvironment) ? DefaultConnectionString : fromEnvironment;
}
}

15
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Models/CentralityScoreEntity.cs Normal file
View File

@@ -0,0 +1,15 @@
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Models;
/// <summary>
/// EF Core entity for graph.centrality_scores table.
/// </summary>
public partial class CentralityScoreEntity
{
public string Tenant { get; set; } = null!;
public string SnapshotId { get; set; } = null!;
public string NodeId { get; set; } = null!;
public double Degree { get; set; }
public double Betweenness { get; set; }
public string Kind { get; set; } = null!;
public DateTimeOffset ComputedAt { get; set; }
}

14
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Models/ClusterAssignmentEntity.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Models;
/// <summary>
/// EF Core entity for graph.cluster_assignments table.
/// </summary>
public partial class ClusterAssignmentEntity
{
public string Tenant { get; set; } = null!;
public string SnapshotId { get; set; } = null!;
public string NodeId { get; set; } = null!;
public string ClusterId { get; set; } = null!;
public string Kind { get; set; } = null!;
public DateTimeOffset ComputedAt { get; set; }
}

14
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Models/GraphEdge.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Models;
/// <summary>
/// EF Core entity for graph.graph_edges table.
/// </summary>
public partial class GraphEdge
{
public string Id { get; set; } = null!;
public string BatchId { get; set; } = null!;
public string SourceId { get; set; } = null!;
public string TargetId { get; set; } = null!;
public string DocumentJson { get; set; } = null!;
public DateTimeOffset WrittenAt { get; set; }
}

12
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Models/GraphNode.cs Normal file
View File

@@ -0,0 +1,12 @@
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Models;
/// <summary>
/// EF Core entity for graph.graph_nodes table.
/// </summary>
public partial class GraphNode
{
public string Id { get; set; } = null!;
public string BatchId { get; set; } = null!;
public string DocumentJson { get; set; } = null!;
public DateTimeOffset WrittenAt { get; set; }
}

10
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Models/IdempotencyToken.cs Normal file
View File

@@ -0,0 +1,10 @@
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Models;
/// <summary>
/// EF Core entity for graph.idempotency_tokens table.
/// </summary>
public partial class IdempotencyToken
{
public string SequenceToken { get; set; } = null!;
public DateTimeOffset SeenAt { get; set; }
}

14
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/EfCore/Models/PendingSnapshot.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Models;
/// <summary>
/// EF Core entity for graph.pending_snapshots table.
/// </summary>
public partial class PendingSnapshot
{
public string Tenant { get; set; } = null!;
public string SnapshotId { get; set; } = null!;
public DateTimeOffset GeneratedAt { get; set; }
public string NodesJson { get; set; } = null!;
public string EdgesJson { get; set; } = null!;
public DateTimeOffset QueuedAt { get; set; }
}

101
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/Migrations/002_efcore_repository_tables.sql Normal file
View File

@@ -0,0 +1,101 @@
-- Graph Indexer Schema Migration 002: EF Core Repository Tables
-- Creates schema-qualified tables used by the EF Core-backed repositories.
-- These tables were previously self-provisioned by repository EnsureTableAsync methods;
-- this migration makes them first-class migration-managed tables.
CREATE SCHEMA IF NOT EXISTS graph;
-- ============================================================================
-- Graph Nodes (schema-qualified, used by PostgresGraphDocumentWriter)
-- ============================================================================
CREATE TABLE IF NOT EXISTS graph.graph_nodes (
id TEXT PRIMARY KEY,
batch_id TEXT NOT NULL,
document_json JSONB NOT NULL,
written_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_graph_nodes_batch_id ON graph.graph_nodes (batch_id);
CREATE INDEX IF NOT EXISTS idx_graph_nodes_written_at ON graph.graph_nodes (written_at);
-- ============================================================================
-- Graph Edges (schema-qualified, used by PostgresGraphDocumentWriter)
-- ============================================================================
CREATE TABLE IF NOT EXISTS graph.graph_edges (
id TEXT PRIMARY KEY,
batch_id TEXT NOT NULL,
source_id TEXT NOT NULL,
target_id TEXT NOT NULL,
document_json JSONB NOT NULL,
written_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_graph_edges_batch_id ON graph.graph_edges (batch_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id ON graph.graph_edges (source_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id ON graph.graph_edges (target_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_written_at ON graph.graph_edges (written_at);
-- ============================================================================
-- Pending Snapshots (used by PostgresGraphSnapshotProvider)
-- ============================================================================
CREATE TABLE IF NOT EXISTS graph.pending_snapshots (
tenant TEXT NOT NULL,
snapshot_id TEXT NOT NULL,
generated_at TIMESTAMPTZ NOT NULL,
nodes_json JSONB NOT NULL,
edges_json JSONB NOT NULL,
queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (tenant, snapshot_id)
);
CREATE INDEX IF NOT EXISTS idx_pending_snapshots_queued_at ON graph.pending_snapshots (queued_at);
-- ============================================================================
-- Cluster Assignments (used by PostgresGraphAnalyticsWriter)
-- ============================================================================
CREATE TABLE IF NOT EXISTS graph.cluster_assignments (
tenant TEXT NOT NULL,
snapshot_id TEXT NOT NULL,
node_id TEXT NOT NULL,
cluster_id TEXT NOT NULL,
kind TEXT NOT NULL,
computed_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (tenant, snapshot_id, node_id)
);
CREATE INDEX IF NOT EXISTS idx_cluster_assignments_cluster ON graph.cluster_assignments (tenant, cluster_id);
CREATE INDEX IF NOT EXISTS idx_cluster_assignments_computed_at ON graph.cluster_assignments (computed_at);
-- ============================================================================
-- Centrality Scores (used by PostgresGraphAnalyticsWriter)
-- ============================================================================
CREATE TABLE IF NOT EXISTS graph.centrality_scores (
tenant TEXT NOT NULL,
snapshot_id TEXT NOT NULL,
node_id TEXT NOT NULL,
degree DOUBLE PRECISION NOT NULL,
betweenness DOUBLE PRECISION NOT NULL,
kind TEXT NOT NULL,
computed_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (tenant, snapshot_id, node_id)
);
CREATE INDEX IF NOT EXISTS idx_centrality_scores_degree ON graph.centrality_scores (tenant, degree DESC);
CREATE INDEX IF NOT EXISTS idx_centrality_scores_betweenness ON graph.centrality_scores (tenant, betweenness DESC);
CREATE INDEX IF NOT EXISTS idx_centrality_scores_computed_at ON graph.centrality_scores (computed_at);
-- ============================================================================
-- Idempotency Tokens (used by PostgresIdempotencyStore)
-- ============================================================================
CREATE TABLE IF NOT EXISTS graph.idempotency_tokens (
sequence_token TEXT PRIMARY KEY,
seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_seen_at ON graph.idempotency_tokens (seen_at);

51
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/Postgres/GraphIndexerDbContextFactory.cs Normal file
View File

@@ -0,0 +1,51 @@
using Microsoft.EntityFrameworkCore;
using Npgsql;
using StellaOps.Graph.Indexer.Persistence.EfCore.CompiledModels;
using StellaOps.Graph.Indexer.Persistence.EfCore.Context;
namespace StellaOps.Graph.Indexer.Persistence.Postgres;
/// <summary>
/// Runtime factory for creating <see cref="GraphIndexerDbContext"/> instances.
/// Uses the static compiled model when schema matches the default and the model is
/// fully initialized; falls back to reflection-based model building otherwise.
/// </summary>
internal static class GraphIndexerDbContextFactory
{
// The compiled model is only usable after `dotnet ef dbcontext optimize` has been run
// against a provisioned database. Until then the stub model contains zero entity types
// and would cause "type is not included in the model" exceptions on every DbSet access.
// We detect a usable model by checking whether it has at least one entity type.
private static readonly bool s_compiledModelUsable = IsCompiledModelUsable();
public static GraphIndexerDbContext Create(NpgsqlConnection connection, int commandTimeoutSeconds, string schemaName)
{
var normalizedSchema = string.IsNullOrWhiteSpace(schemaName)
? GraphIndexerDataSource.DefaultSchemaName
: schemaName.Trim();
var optionsBuilder = new DbContextOptionsBuilder<GraphIndexerDbContext>()
.UseNpgsql(connection, npgsql => npgsql.CommandTimeout(commandTimeoutSeconds));
if (s_compiledModelUsable &&
string.Equals(normalizedSchema, GraphIndexerDataSource.DefaultSchemaName, StringComparison.Ordinal))
{
optionsBuilder.UseModel(GraphIndexerDbContextModel.Instance);
}
return new GraphIndexerDbContext(optionsBuilder.Options, normalizedSchema);
}
private static bool IsCompiledModelUsable()
{
try
{
var model = GraphIndexerDbContextModel.Instance;
return model.GetEntityTypes().Any();
}
catch
{
return false;
}
}
}

153
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/Postgres/Repositories/PostgresGraphAnalyticsWriter.cs
View File

@@ -1,18 +1,19 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Graph.Indexer.Analytics;
using StellaOps.Graph.Indexer.Persistence.EfCore.Models;
using StellaOps.Infrastructure.Postgres.Repositories;
using System.Collections.Immutable;
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
/// <summary>
/// PostgreSQL implementation of <see cref="IGraphAnalyticsWriter"/>.
/// PostgreSQL (EF Core) implementation of <see cref="IGraphAnalyticsWriter"/>.
/// </summary>
public sealed class PostgresGraphAnalyticsWriter : RepositoryBase<GraphIndexerDataSource>, IGraphAnalyticsWriter
{
private bool _tableInitialized;
private const int WriteCommandTimeoutSeconds = 60;
public PostgresGraphAnalyticsWriter(GraphIndexerDataSource dataSource, ILogger<PostgresGraphAnalyticsWriter> logger)
: base(dataSource, logger)
@@ -26,44 +27,35 @@ public sealed class PostgresGraphAnalyticsWriter : RepositoryBase<GraphIndexerDa
{
ArgumentNullException.ThrowIfNull(snapshot);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, WriteCommandTimeoutSeconds, GetSchemaName());
await using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
try
{
// Delete existing assignments for this snapshot
const string deleteSql = @"
DELETE FROM graph.cluster_assignments
WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
var tenant = snapshot.Tenant ?? string.Empty;
var snapshotId = snapshot.SnapshotId ?? string.Empty;
await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
{
AddParameter(deleteCommand, "@tenant", snapshot.Tenant ?? string.Empty);
AddParameter(deleteCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
// Delete existing assignments for this snapshot
await dbContext.ClusterAssignments
.Where(ca => ca.Tenant == tenant && ca.SnapshotId == snapshotId)
.ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
// Insert new assignments
const string insertSql = @"
INSERT INTO graph.cluster_assignments (tenant, snapshot_id, node_id, cluster_id, kind, computed_at)
VALUES (@tenant, @snapshot_id, @node_id, @cluster_id, @kind, @computed_at)";
var computedAt = snapshot.GeneratedAt;
foreach (var assignment in assignments)
var entities = assignments.Select(assignment => new ClusterAssignmentEntity
{
await using var insertCommand = CreateCommand(insertSql, connection, transaction);
AddParameter(insertCommand, "@tenant", snapshot.Tenant ?? string.Empty);
AddParameter(insertCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
AddParameter(insertCommand, "@node_id", assignment.NodeId ?? string.Empty);
AddParameter(insertCommand, "@cluster_id", assignment.ClusterId ?? string.Empty);
AddParameter(insertCommand, "@kind", assignment.Kind ?? string.Empty);
AddParameter(insertCommand, "@computed_at", computedAt);
Tenant = tenant,
SnapshotId = snapshotId,
NodeId = assignment.NodeId ?? string.Empty,
ClusterId = assignment.ClusterId ?? string.Empty,
Kind = assignment.Kind ?? string.Empty,
ComputedAt = computedAt
});
await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
dbContext.ClusterAssignments.AddRange(entities);
await dbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
}
@@ -81,45 +73,36 @@ public sealed class PostgresGraphAnalyticsWriter : RepositoryBase<GraphIndexerDa
{
ArgumentNullException.ThrowIfNull(snapshot);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, WriteCommandTimeoutSeconds, GetSchemaName());
await using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
try
{
// Delete existing scores for this snapshot
const string deleteSql = @"
DELETE FROM graph.centrality_scores
WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
var tenant = snapshot.Tenant ?? string.Empty;
var snapshotId = snapshot.SnapshotId ?? string.Empty;
await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
{
AddParameter(deleteCommand, "@tenant", snapshot.Tenant ?? string.Empty);
AddParameter(deleteCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
// Delete existing scores for this snapshot
await dbContext.CentralityScores
.Where(cs => cs.Tenant == tenant && cs.SnapshotId == snapshotId)
.ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
// Insert new scores
const string insertSql = @"
INSERT INTO graph.centrality_scores (tenant, snapshot_id, node_id, degree, betweenness, kind, computed_at)
VALUES (@tenant, @snapshot_id, @node_id, @degree, @betweenness, @kind, @computed_at)";
var computedAt = snapshot.GeneratedAt;
foreach (var score in scores)
var entities = scores.Select(score => new CentralityScoreEntity
{
await using var insertCommand = CreateCommand(insertSql, connection, transaction);
AddParameter(insertCommand, "@tenant", snapshot.Tenant ?? string.Empty);
AddParameter(insertCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
AddParameter(insertCommand, "@node_id", score.NodeId ?? string.Empty);
AddParameter(insertCommand, "@degree", score.Degree);
AddParameter(insertCommand, "@betweenness", score.Betweenness);
AddParameter(insertCommand, "@kind", score.Kind ?? string.Empty);
AddParameter(insertCommand, "@computed_at", computedAt);
Tenant = tenant,
SnapshotId = snapshotId,
NodeId = score.NodeId ?? string.Empty,
Degree = score.Degree,
Betweenness = score.Betweenness,
Kind = score.Kind ?? string.Empty,
ComputedAt = computedAt
});
await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
dbContext.CentralityScores.AddRange(entities);
await dbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
}
@@ -130,53 +113,5 @@ public sealed class PostgresGraphAnalyticsWriter : RepositoryBase<GraphIndexerDa
}
}
private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
{
return new NpgsqlCommand(sql, connection, transaction);
}
private async Task EnsureTableAsync(CancellationToken cancellationToken)
{
if (_tableInitialized)
{
return;
}
const string ddl = @"
CREATE SCHEMA IF NOT EXISTS graph;
CREATE TABLE IF NOT EXISTS graph.cluster_assignments (
tenant TEXT NOT NULL,
snapshot_id TEXT NOT NULL,
node_id TEXT NOT NULL,
cluster_id TEXT NOT NULL,
kind TEXT NOT NULL,
computed_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (tenant, snapshot_id, node_id)
);
CREATE INDEX IF NOT EXISTS idx_cluster_assignments_cluster ON graph.cluster_assignments (tenant, cluster_id);
CREATE INDEX IF NOT EXISTS idx_cluster_assignments_computed_at ON graph.cluster_assignments (computed_at);
CREATE TABLE IF NOT EXISTS graph.centrality_scores (
tenant TEXT NOT NULL,
snapshot_id TEXT NOT NULL,
node_id TEXT NOT NULL,
degree DOUBLE PRECISION NOT NULL,
betweenness DOUBLE PRECISION NOT NULL,
kind TEXT NOT NULL,
computed_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (tenant, snapshot_id, node_id)
);
CREATE INDEX IF NOT EXISTS idx_centrality_scores_degree ON graph.centrality_scores (tenant, degree DESC);
CREATE INDEX IF NOT EXISTS idx_centrality_scores_betweenness ON graph.centrality_scores (tenant, betweenness DESC);
CREATE INDEX IF NOT EXISTS idx_centrality_scores_computed_at ON graph.centrality_scores (computed_at);";
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(ddl, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_tableInitialized = true;
}
private static string GetSchemaName() => GraphIndexerDataSource.DefaultSchemaName;
}

103
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/Postgres/Repositories/PostgresGraphDocumentWriter.cs
View File

@@ -1,8 +1,9 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Determinism;
using StellaOps.Graph.Indexer.Ingestion.Sbom;
using StellaOps.Graph.Indexer.Persistence.EfCore.Models;
using StellaOps.Infrastructure.Postgres.Repositories;
using System.Text.Json;
using System.Text.Json.Nodes;
@@ -10,7 +11,7 @@ using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
/// <summary>
/// PostgreSQL implementation of <see cref="IGraphDocumentWriter"/>.
/// PostgreSQL (EF Core) implementation of <see cref="IGraphDocumentWriter"/>.
/// </summary>
public sealed class PostgresGraphDocumentWriter : RepositoryBase<GraphIndexerDataSource>, IGraphDocumentWriter
{
@@ -21,7 +22,7 @@ public sealed class PostgresGraphDocumentWriter : RepositoryBase<GraphIndexerDat
};
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider; private bool _tableInitialized;
private readonly IGuidProvider _guidProvider;
public PostgresGraphDocumentWriter(
GraphIndexerDataSource dataSource,
@@ -38,64 +39,56 @@ public sealed class PostgresGraphDocumentWriter : RepositoryBase<GraphIndexerDat
{
ArgumentNullException.ThrowIfNull(batch);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
await using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
try
{
var batchId = _guidProvider.NewGuid().ToString("N");
var writtenAt = _timeProvider.GetUtcNow();
// Insert nodes
// Upsert nodes via raw SQL for ON CONFLICT DO UPDATE
foreach (var node in batch.Nodes)
{
var nodeId = ExtractId(node);
var nodeJson = node.ToJsonString();
const string nodeSql = @"
await dbContext.Database.ExecuteSqlRawAsync(
"""
INSERT INTO graph.graph_nodes (id, batch_id, document_json, written_at)
VALUES (@id, @batch_id, @document_json, @written_at)
VALUES ({0}, {1}, {2}::jsonb, {3})
ON CONFLICT (id) DO UPDATE SET
batch_id = EXCLUDED.batch_id,
document_json = EXCLUDED.document_json,
written_at = EXCLUDED.written_at";
await using var nodeCommand = CreateCommand(nodeSql, connection, transaction);
AddParameter(nodeCommand, "@id", nodeId);
AddParameter(nodeCommand, "@batch_id", batchId);
AddJsonbParameter(nodeCommand, "@document_json", nodeJson);
AddParameter(nodeCommand, "@written_at", writtenAt);
await nodeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
written_at = EXCLUDED.written_at
""",
[nodeId, batchId, nodeJson, writtenAt],
cancellationToken).ConfigureAwait(false);
}
// Insert edges
// Upsert edges via raw SQL for ON CONFLICT DO UPDATE
foreach (var edge in batch.Edges)
{
var edgeId = ExtractEdgeId(edge);
var edgeJson = edge.ToJsonString();
var sourceId = ExtractString(edge, "source") ?? string.Empty;
var targetId = ExtractString(edge, "target") ?? string.Empty;
const string edgeSql = @"
await dbContext.Database.ExecuteSqlRawAsync(
"""
INSERT INTO graph.graph_edges (id, batch_id, source_id, target_id, document_json, written_at)
VALUES (@id, @batch_id, @source_id, @target_id, @document_json, @written_at)
VALUES ({0}, {1}, {2}, {3}, {4}::jsonb, {5})
ON CONFLICT (id) DO UPDATE SET
batch_id = EXCLUDED.batch_id,
source_id = EXCLUDED.source_id,
target_id = EXCLUDED.target_id,
document_json = EXCLUDED.document_json,
written_at = EXCLUDED.written_at";
await using var edgeCommand = CreateCommand(edgeSql, connection, transaction);
AddParameter(edgeCommand, "@id", edgeId);
AddParameter(edgeCommand, "@batch_id", batchId);
AddParameter(edgeCommand, "@source_id", ExtractString(edge, "source") ?? string.Empty);
AddParameter(edgeCommand, "@target_id", ExtractString(edge, "target") ?? string.Empty);
AddJsonbParameter(edgeCommand, "@document_json", edgeJson);
AddParameter(edgeCommand, "@written_at", writtenAt);
await edgeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
written_at = EXCLUDED.written_at
""",
[edgeId, batchId, sourceId, targetId, edgeJson, writtenAt],
cancellationToken).ConfigureAwait(false);
}
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
@@ -135,49 +128,5 @@ public sealed class PostgresGraphDocumentWriter : RepositoryBase<GraphIndexerDat
return null;
}
private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
{
return new NpgsqlCommand(sql, connection, transaction);
}
private async Task EnsureTableAsync(CancellationToken cancellationToken)
{
if (_tableInitialized)
{
return;
}
const string ddl = @"
CREATE SCHEMA IF NOT EXISTS graph;
CREATE TABLE IF NOT EXISTS graph.graph_nodes (
id TEXT PRIMARY KEY,
batch_id TEXT NOT NULL,
document_json JSONB NOT NULL,
written_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_graph_nodes_batch_id ON graph.graph_nodes (batch_id);
CREATE INDEX IF NOT EXISTS idx_graph_nodes_written_at ON graph.graph_nodes (written_at);
CREATE TABLE IF NOT EXISTS graph.graph_edges (
id TEXT PRIMARY KEY,
batch_id TEXT NOT NULL,
source_id TEXT NOT NULL,
target_id TEXT NOT NULL,
document_json JSONB NOT NULL,
written_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_graph_edges_batch_id ON graph.graph_edges (batch_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id ON graph.graph_edges (source_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id ON graph.graph_edges (target_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_written_at ON graph.graph_edges (written_at);";
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(ddl, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_tableInitialized = true;
}
private static string GetSchemaName() => GraphIndexerDataSource.DefaultSchemaName;
}

121
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/Postgres/Repositories/PostgresGraphSnapshotProvider.cs
View File

@@ -1,7 +1,8 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Graph.Indexer.Analytics;
using StellaOps.Graph.Indexer.Persistence.EfCore.Models;
using StellaOps.Infrastructure.Postgres.Repositories;
using System.Collections.Immutable;
using System.Text.Json;
@@ -10,7 +11,7 @@ using System.Text.Json.Nodes;
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
/// <summary>
/// PostgreSQL implementation of <see cref="IGraphSnapshotProvider"/>.
/// PostgreSQL (EF Core) implementation of <see cref="IGraphSnapshotProvider"/>.
/// </summary>
public sealed class PostgresGraphSnapshotProvider : RepositoryBase<GraphIndexerDataSource>, IGraphSnapshotProvider
{
@@ -20,7 +21,6 @@ public sealed class PostgresGraphSnapshotProvider : RepositoryBase<GraphIndexerD
WriteIndented = false
};
private bool _tableInitialized;
private readonly TimeProvider _timeProvider;
public PostgresGraphSnapshotProvider(
@@ -39,83 +39,63 @@ public sealed class PostgresGraphSnapshotProvider : RepositoryBase<GraphIndexerD
{
ArgumentNullException.ThrowIfNull(snapshot);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
var nodesJson = JsonSerializer.Serialize(snapshot.Nodes.Select(n => n.ToJsonString()), JsonOptions);
var edgesJson = JsonSerializer.Serialize(snapshot.Edges.Select(e => e.ToJsonString()), JsonOptions);
var tenant = snapshot.Tenant ?? string.Empty;
var snapshotId = snapshot.SnapshotId ?? string.Empty;
var queuedAt = _timeProvider.GetUtcNow();
const string sql = @"
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
// Use raw SQL for upsert ON CONFLICT pattern
await dbContext.Database.ExecuteSqlRawAsync(
"""
INSERT INTO graph.pending_snapshots (tenant, snapshot_id, generated_at, nodes_json, edges_json, queued_at)
VALUES (@tenant, @snapshot_id, @generated_at, @nodes_json, @edges_json, @queued_at)
VALUES ({0}, {1}, {2}, {3}::jsonb, {4}::jsonb, {5})
ON CONFLICT (tenant, snapshot_id) DO UPDATE SET
generated_at = EXCLUDED.generated_at,
nodes_json = EXCLUDED.nodes_json,
edges_json = EXCLUDED.edges_json,
queued_at = EXCLUDED.queued_at";
var nodesJson = JsonSerializer.Serialize(snapshot.Nodes.Select(n => n.ToJsonString()), JsonOptions);
var edgesJson = JsonSerializer.Serialize(snapshot.Edges.Select(e => e.ToJsonString()), JsonOptions);
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(sql, connection);
AddParameter(command, "@tenant", snapshot.Tenant ?? string.Empty);
AddParameter(command, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
AddParameter(command, "@generated_at", snapshot.GeneratedAt);
AddJsonbParameter(command, "@nodes_json", nodesJson);
AddJsonbParameter(command, "@edges_json", edgesJson);
AddParameter(command, "@queued_at", _timeProvider.GetUtcNow());
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
queued_at = EXCLUDED.queued_at
""",
[tenant, snapshotId, snapshot.GeneratedAt, nodesJson, edgesJson, queuedAt],
cancellationToken).ConfigureAwait(false);
}
public async Task<IReadOnlyList<GraphAnalyticsSnapshot>> GetPendingSnapshotsAsync(CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
const string sql = @"
SELECT tenant, snapshot_id, generated_at, nodes_json, edges_json
FROM graph.pending_snapshots
ORDER BY queued_at ASC
LIMIT 100";
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(sql, connection);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
var results = new List<GraphAnalyticsSnapshot>();
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
results.Add(MapSnapshot(reader));
}
var entities = await dbContext.PendingSnapshots
.AsNoTracking()
.OrderBy(ps => ps.QueuedAt)
.Take(100)
.ToListAsync(cancellationToken).ConfigureAwait(false);
return results.ToImmutableArray();
return entities.Select(MapSnapshot).ToImmutableArray();
}
public async Task MarkProcessedAsync(string tenant, string snapshotId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(snapshotId);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
const string sql = @"
DELETE FROM graph.pending_snapshots
WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
var normalizedTenant = tenant ?? string.Empty;
var normalizedSnapshotId = snapshotId.Trim();
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(sql, connection);
AddParameter(command, "@tenant", tenant ?? string.Empty);
AddParameter(command, "@snapshot_id", snapshotId.Trim());
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
await dbContext.PendingSnapshots
.Where(ps => ps.Tenant == normalizedTenant && ps.SnapshotId == normalizedSnapshotId)
.ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
}
private static GraphAnalyticsSnapshot MapSnapshot(NpgsqlDataReader reader)
private static GraphAnalyticsSnapshot MapSnapshot(PendingSnapshot entity)
{
var tenant = reader.GetString(0);
var snapshotId = reader.GetString(1);
var generatedAt = reader.GetFieldValue<DateTimeOffset>(2);
var nodesJson = reader.GetString(3);
var edgesJson = reader.GetString(4);
var nodeStrings = JsonSerializer.Deserialize<List<string>>(nodesJson, JsonOptions) ?? new List<string>();
var edgeStrings = JsonSerializer.Deserialize<List<string>>(edgesJson, JsonOptions) ?? new List<string>();
var nodeStrings = JsonSerializer.Deserialize<List<string>>(entity.NodesJson, JsonOptions) ?? new List<string>();
var edgeStrings = JsonSerializer.Deserialize<List<string>>(entity.EdgesJson, JsonOptions) ?? new List<string>();
var nodes = nodeStrings
.Select(s => JsonNode.Parse(s) as JsonObject)
@@ -129,35 +109,8 @@ public sealed class PostgresGraphSnapshotProvider : RepositoryBase<GraphIndexerD
.Cast<JsonObject>()
.ToImmutableArray();
return new GraphAnalyticsSnapshot(tenant, snapshotId, generatedAt, nodes, edges);
return new GraphAnalyticsSnapshot(entity.Tenant, entity.SnapshotId, entity.GeneratedAt, nodes, edges);
}
private async Task EnsureTableAsync(CancellationToken cancellationToken)
{
if (_tableInitialized)
{
return;
}
const string ddl = @"
CREATE SCHEMA IF NOT EXISTS graph;
CREATE TABLE IF NOT EXISTS graph.pending_snapshots (
tenant TEXT NOT NULL,
snapshot_id TEXT NOT NULL,
generated_at TIMESTAMPTZ NOT NULL,
nodes_json JSONB NOT NULL,
edges_json JSONB NOT NULL,
queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (tenant, snapshot_id)
);
CREATE INDEX IF NOT EXISTS idx_pending_snapshots_queued_at ON graph.pending_snapshots (queued_at);";
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(ddl, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_tableInitialized = true;
}
private static string GetSchemaName() => GraphIndexerDataSource.DefaultSchemaName;
}

68
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/Postgres/Repositories/PostgresIdempotencyStore.cs
View File

@@ -1,16 +1,17 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Graph.Indexer.Incremental;
using StellaOps.Graph.Indexer.Persistence.EfCore.Models;
using StellaOps.Infrastructure.Postgres.Repositories;
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
/// <summary>
/// PostgreSQL implementation of <see cref="IIdempotencyStore"/>.
/// PostgreSQL (EF Core) implementation of <see cref="IIdempotencyStore"/>.
/// </summary>
public sealed class PostgresIdempotencyStore : RepositoryBase<GraphIndexerDataSource>, IIdempotencyStore
{
private bool _tableInitialized;
public PostgresIdempotencyStore(GraphIndexerDataSource dataSource, ILogger<PostgresIdempotencyStore> logger)
: base(dataSource, logger)
{
@@ -20,59 +21,36 @@ public sealed class PostgresIdempotencyStore : RepositoryBase<GraphIndexerDataSo
{
ArgumentException.ThrowIfNullOrWhiteSpace(sequenceToken);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
const string sql = @"
SELECT EXISTS(SELECT 1 FROM graph.idempotency_tokens WHERE sequence_token = @sequence_token)";
var normalizedToken = sequenceToken.Trim();
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(sql, connection);
AddParameter(command, "@sequence_token", sequenceToken.Trim());
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return result is bool seen && seen;
return await dbContext.IdempotencyTokens
.AsNoTracking()
.AnyAsync(t => t.SequenceToken == normalizedToken, cancellationToken).ConfigureAwait(false);
}
public async Task MarkSeenAsync(string sequenceToken, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sequenceToken);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
var normalizedToken = sequenceToken.Trim();
var seenAt = DateTimeOffset.UtcNow;
const string sql = @"
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = GraphIndexerDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
// Use raw SQL for upsert ON CONFLICT DO NOTHING pattern (idempotent)
await dbContext.Database.ExecuteSqlRawAsync(
"""
INSERT INTO graph.idempotency_tokens (sequence_token, seen_at)
VALUES (@sequence_token, @seen_at)
ON CONFLICT (sequence_token) DO NOTHING";
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(sql, connection);
AddParameter(command, "@sequence_token", sequenceToken.Trim());
AddParameter(command, "@seen_at", DateTimeOffset.UtcNow);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
VALUES ({0}, {1})
ON CONFLICT (sequence_token) DO NOTHING
""",
[normalizedToken, seenAt],
cancellationToken).ConfigureAwait(false);
}
private async Task EnsureTableAsync(CancellationToken cancellationToken)
{
if (_tableInitialized)
{
return;
}
const string ddl = @"
CREATE SCHEMA IF NOT EXISTS graph;
CREATE TABLE IF NOT EXISTS graph.idempotency_tokens (
sequence_token TEXT PRIMARY KEY,
seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_seen_at ON graph.idempotency_tokens (seen_at);";
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var command = CreateCommand(ddl, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_tableInitialized = true;
}
private static string GetSchemaName() => GraphIndexerDataSource.DefaultSchemaName;
}

7
src/Graph/__Libraries/StellaOps.Graph.Indexer.Persistence/StellaOps.Graph.Indexer.Persistence.csproj
View File

@@ -11,7 +11,12 @@
</PropertyGroup>
<ItemGroup>
<EmbeddedResource Include="Migrations\*.sql" />
<EmbeddedResource Include="Migrations\**\*.sql" LogicalName="%(RecursiveDir)%(Filename)%(Extension)" />
</ItemGroup>
<ItemGroup>
<!-- Prevent automatic compiled-model binding so non-default schemas can build runtime models. -->
<Compile Remove="EfCore\CompiledModels\GraphIndexerDbContextAssemblyAttributes.cs" />
</ItemGroup>
<ItemGroup>