Fix build and code structure improvements. New but essential UI functionality. CI improvements. Documentation improvements. AI module improvements.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,21 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.EfCore.Context;
|
||||
|
||||
/// <summary>
|
||||
/// EF Core DbContext for Graph Indexer module.
|
||||
/// This is a stub that will be scaffolded from the PostgreSQL database.
|
||||
/// </summary>
|
||||
public class GraphIndexerDbContext : DbContext
|
||||
{
|
||||
public GraphIndexerDbContext(DbContextOptions<GraphIndexerDbContext> options)
|
||||
: base(options)
|
||||
{
|
||||
}
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.HasDefaultSchema("graph");
|
||||
base.OnModelCreating(modelBuilder);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Graph.Indexer.Analytics;
|
||||
using StellaOps.Graph.Indexer.Incremental;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Graph.Indexer.Persistence.Postgres;
|
||||
using StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.Extensions;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for configuring Graph.Indexer persistence services.
|
||||
/// </summary>
|
||||
public static class GraphIndexerPersistenceExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds Graph.Indexer PostgreSQL persistence services.
|
||||
/// </summary>
|
||||
public static IServiceCollection AddGraphIndexerPersistence(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "Postgres:Graph")
|
||||
{
|
||||
services.Configure<PostgresOptions>(configuration.GetSection(sectionName));
|
||||
services.AddSingleton<GraphIndexerDataSource>();
|
||||
|
||||
// Register repositories
|
||||
services.AddSingleton<IIdempotencyStore, PostgresIdempotencyStore>();
|
||||
services.AddSingleton<IGraphSnapshotProvider, PostgresGraphSnapshotProvider>();
|
||||
services.AddSingleton<IGraphAnalyticsWriter, PostgresGraphAnalyticsWriter>();
|
||||
services.AddSingleton<IGraphDocumentWriter, PostgresGraphDocumentWriter>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds Graph.Indexer PostgreSQL persistence services with explicit options.
|
||||
/// </summary>
|
||||
public static IServiceCollection AddGraphIndexerPersistence(
|
||||
this IServiceCollection services,
|
||||
Action<PostgresOptions> configureOptions)
|
||||
{
|
||||
services.Configure(configureOptions);
|
||||
services.AddSingleton<GraphIndexerDataSource>();
|
||||
|
||||
// Register repositories
|
||||
services.AddSingleton<IIdempotencyStore, PostgresIdempotencyStore>();
|
||||
services.AddSingleton<IGraphSnapshotProvider, PostgresGraphSnapshotProvider>();
|
||||
services.AddSingleton<IGraphAnalyticsWriter, PostgresGraphAnalyticsWriter>();
|
||||
services.AddSingleton<IGraphDocumentWriter, PostgresGraphDocumentWriter>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Connections;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.Postgres;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL data source for Graph.Indexer module.
|
||||
/// </summary>
|
||||
public sealed class GraphIndexerDataSource : DataSourceBase
|
||||
{
|
||||
/// <summary>
|
||||
/// Default schema name for Graph.Indexer tables.
|
||||
/// </summary>
|
||||
public const string DefaultSchemaName = "graph";
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new Graph.Indexer data source.
|
||||
/// </summary>
|
||||
public GraphIndexerDataSource(IOptions<PostgresOptions> options, ILogger<GraphIndexerDataSource> logger)
|
||||
: base(CreateOptions(options.Value), logger)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override string ModuleName => "Graph.Indexer";
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void ConfigureDataSourceBuilder(NpgsqlDataSourceBuilder builder)
|
||||
{
|
||||
base.ConfigureDataSourceBuilder(builder);
|
||||
}
|
||||
|
||||
private static PostgresOptions CreateOptions(PostgresOptions baseOptions)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(baseOptions.SchemaName))
|
||||
{
|
||||
baseOptions.SchemaName = DefaultSchemaName;
|
||||
}
|
||||
return baseOptions;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using System.Collections.Immutable;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Graph.Indexer.Analytics;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IGraphAnalyticsWriter"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresGraphAnalyticsWriter : RepositoryBase<GraphIndexerDataSource>, IGraphAnalyticsWriter
|
||||
{
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresGraphAnalyticsWriter(GraphIndexerDataSource dataSource, ILogger<PostgresGraphAnalyticsWriter> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task PersistClusterAssignmentsAsync(
|
||||
GraphAnalyticsSnapshot snapshot,
|
||||
ImmutableArray<ClusterAssignment> assignments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
// Delete existing assignments for this snapshot
|
||||
const string deleteSql = @"
|
||||
DELETE FROM graph.cluster_assignments
|
||||
WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
|
||||
|
||||
await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
|
||||
{
|
||||
AddParameter(deleteCommand, "@tenant", snapshot.Tenant ?? string.Empty);
|
||||
AddParameter(deleteCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
|
||||
await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Insert new assignments
|
||||
const string insertSql = @"
|
||||
INSERT INTO graph.cluster_assignments (tenant, snapshot_id, node_id, cluster_id, kind, computed_at)
|
||||
VALUES (@tenant, @snapshot_id, @node_id, @cluster_id, @kind, @computed_at)";
|
||||
|
||||
var computedAt = snapshot.GeneratedAt;
|
||||
|
||||
foreach (var assignment in assignments)
|
||||
{
|
||||
await using var insertCommand = CreateCommand(insertSql, connection, transaction);
|
||||
AddParameter(insertCommand, "@tenant", snapshot.Tenant ?? string.Empty);
|
||||
AddParameter(insertCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
|
||||
AddParameter(insertCommand, "@node_id", assignment.NodeId ?? string.Empty);
|
||||
AddParameter(insertCommand, "@cluster_id", assignment.ClusterId ?? string.Empty);
|
||||
AddParameter(insertCommand, "@kind", assignment.Kind ?? string.Empty);
|
||||
AddParameter(insertCommand, "@computed_at", computedAt);
|
||||
|
||||
await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task PersistCentralityAsync(
|
||||
GraphAnalyticsSnapshot snapshot,
|
||||
ImmutableArray<CentralityScore> scores,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
// Delete existing scores for this snapshot
|
||||
const string deleteSql = @"
|
||||
DELETE FROM graph.centrality_scores
|
||||
WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
|
||||
|
||||
await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
|
||||
{
|
||||
AddParameter(deleteCommand, "@tenant", snapshot.Tenant ?? string.Empty);
|
||||
AddParameter(deleteCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
|
||||
await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Insert new scores
|
||||
const string insertSql = @"
|
||||
INSERT INTO graph.centrality_scores (tenant, snapshot_id, node_id, degree, betweenness, kind, computed_at)
|
||||
VALUES (@tenant, @snapshot_id, @node_id, @degree, @betweenness, @kind, @computed_at)";
|
||||
|
||||
var computedAt = snapshot.GeneratedAt;
|
||||
|
||||
foreach (var score in scores)
|
||||
{
|
||||
await using var insertCommand = CreateCommand(insertSql, connection, transaction);
|
||||
AddParameter(insertCommand, "@tenant", snapshot.Tenant ?? string.Empty);
|
||||
AddParameter(insertCommand, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
|
||||
AddParameter(insertCommand, "@node_id", score.NodeId ?? string.Empty);
|
||||
AddParameter(insertCommand, "@degree", score.Degree);
|
||||
AddParameter(insertCommand, "@betweenness", score.Betweenness);
|
||||
AddParameter(insertCommand, "@kind", score.Kind ?? string.Empty);
|
||||
AddParameter(insertCommand, "@computed_at", computedAt);
|
||||
|
||||
await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
|
||||
{
|
||||
return new NpgsqlCommand(sql, connection, transaction);
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS graph;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph.cluster_assignments (
|
||||
tenant TEXT NOT NULL,
|
||||
snapshot_id TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
cluster_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
computed_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (tenant, snapshot_id, node_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cluster_assignments_cluster ON graph.cluster_assignments (tenant, cluster_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cluster_assignments_computed_at ON graph.cluster_assignments (computed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph.centrality_scores (
|
||||
tenant TEXT NOT NULL,
|
||||
snapshot_id TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
degree DOUBLE PRECISION NOT NULL,
|
||||
betweenness DOUBLE PRECISION NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
computed_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (tenant, snapshot_id, node_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_centrality_scores_degree ON graph.centrality_scores (tenant, degree DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_centrality_scores_betweenness ON graph.centrality_scores (tenant, betweenness DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_centrality_scores_computed_at ON graph.centrality_scores (computed_at);";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Graph.Indexer.Ingestion.Sbom;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IGraphDocumentWriter"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresGraphDocumentWriter : RepositoryBase<GraphIndexerDataSource>, IGraphDocumentWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresGraphDocumentWriter(GraphIndexerDataSource dataSource, ILogger<PostgresGraphDocumentWriter> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task WriteAsync(GraphBuildBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(batch);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
var batchId = Guid.NewGuid().ToString("N");
|
||||
var writtenAt = DateTimeOffset.UtcNow;
|
||||
|
||||
// Insert nodes
|
||||
foreach (var node in batch.Nodes)
|
||||
{
|
||||
var nodeId = ExtractId(node);
|
||||
var nodeJson = node.ToJsonString();
|
||||
|
||||
const string nodeSql = @"
|
||||
INSERT INTO graph.graph_nodes (id, batch_id, document_json, written_at)
|
||||
VALUES (@id, @batch_id, @document_json, @written_at)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
batch_id = EXCLUDED.batch_id,
|
||||
document_json = EXCLUDED.document_json,
|
||||
written_at = EXCLUDED.written_at";
|
||||
|
||||
await using var nodeCommand = CreateCommand(nodeSql, connection, transaction);
|
||||
AddParameter(nodeCommand, "@id", nodeId);
|
||||
AddParameter(nodeCommand, "@batch_id", batchId);
|
||||
AddJsonbParameter(nodeCommand, "@document_json", nodeJson);
|
||||
AddParameter(nodeCommand, "@written_at", writtenAt);
|
||||
|
||||
await nodeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Insert edges
|
||||
foreach (var edge in batch.Edges)
|
||||
{
|
||||
var edgeId = ExtractEdgeId(edge);
|
||||
var edgeJson = edge.ToJsonString();
|
||||
|
||||
const string edgeSql = @"
|
||||
INSERT INTO graph.graph_edges (id, batch_id, source_id, target_id, document_json, written_at)
|
||||
VALUES (@id, @batch_id, @source_id, @target_id, @document_json, @written_at)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
batch_id = EXCLUDED.batch_id,
|
||||
source_id = EXCLUDED.source_id,
|
||||
target_id = EXCLUDED.target_id,
|
||||
document_json = EXCLUDED.document_json,
|
||||
written_at = EXCLUDED.written_at";
|
||||
|
||||
await using var edgeCommand = CreateCommand(edgeSql, connection, transaction);
|
||||
AddParameter(edgeCommand, "@id", edgeId);
|
||||
AddParameter(edgeCommand, "@batch_id", batchId);
|
||||
AddParameter(edgeCommand, "@source_id", ExtractString(edge, "source") ?? string.Empty);
|
||||
AddParameter(edgeCommand, "@target_id", ExtractString(edge, "target") ?? string.Empty);
|
||||
AddJsonbParameter(edgeCommand, "@document_json", edgeJson);
|
||||
AddParameter(edgeCommand, "@written_at", writtenAt);
|
||||
|
||||
await edgeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static string ExtractId(JsonObject node)
|
||||
{
|
||||
return ExtractString(node, "id") ?? ExtractString(node, "@id") ?? Guid.NewGuid().ToString("N");
|
||||
}
|
||||
|
||||
private static string ExtractEdgeId(JsonObject edge)
|
||||
{
|
||||
var id = ExtractString(edge, "id") ?? ExtractString(edge, "@id");
|
||||
if (!string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
return id;
|
||||
}
|
||||
|
||||
var source = ExtractString(edge, "source") ?? string.Empty;
|
||||
var target = ExtractString(edge, "target") ?? string.Empty;
|
||||
var type = ExtractString(edge, "type") ?? ExtractString(edge, "relationship") ?? "relates_to";
|
||||
return $"{source}|{target}|{type}";
|
||||
}
|
||||
|
||||
private static string? ExtractString(JsonObject obj, string key)
|
||||
{
|
||||
if (obj.TryGetPropertyValue(key, out var value) && value is JsonValue jv)
|
||||
{
|
||||
return jv.GetValue<string>();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
|
||||
{
|
||||
return new NpgsqlCommand(sql, connection, transaction);
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS graph;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph.graph_nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
batch_id TEXT NOT NULL,
|
||||
document_json JSONB NOT NULL,
|
||||
written_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_nodes_batch_id ON graph.graph_nodes (batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_nodes_written_at ON graph.graph_nodes (written_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph.graph_edges (
|
||||
id TEXT PRIMARY KEY,
|
||||
batch_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL,
|
||||
document_json JSONB NOT NULL,
|
||||
written_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_edges_batch_id ON graph.graph_edges (batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id ON graph.graph_edges (source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id ON graph.graph_edges (target_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_edges_written_at ON graph.graph_edges (written_at);";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Graph.Indexer.Analytics;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IGraphSnapshotProvider"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresGraphSnapshotProvider : RepositoryBase<GraphIndexerDataSource>, IGraphSnapshotProvider
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresGraphSnapshotProvider(GraphIndexerDataSource dataSource, ILogger<PostgresGraphSnapshotProvider> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueues a snapshot for processing.
|
||||
/// </summary>
|
||||
public async Task EnqueueAsync(GraphAnalyticsSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
INSERT INTO graph.pending_snapshots (tenant, snapshot_id, generated_at, nodes_json, edges_json, queued_at)
|
||||
VALUES (@tenant, @snapshot_id, @generated_at, @nodes_json, @edges_json, @queued_at)
|
||||
ON CONFLICT (tenant, snapshot_id) DO UPDATE SET
|
||||
generated_at = EXCLUDED.generated_at,
|
||||
nodes_json = EXCLUDED.nodes_json,
|
||||
edges_json = EXCLUDED.edges_json,
|
||||
queued_at = EXCLUDED.queued_at";
|
||||
|
||||
var nodesJson = JsonSerializer.Serialize(snapshot.Nodes.Select(n => n.ToJsonString()), JsonOptions);
|
||||
var edgesJson = JsonSerializer.Serialize(snapshot.Edges.Select(e => e.ToJsonString()), JsonOptions);
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@tenant", snapshot.Tenant ?? string.Empty);
|
||||
AddParameter(command, "@snapshot_id", snapshot.SnapshotId ?? string.Empty);
|
||||
AddParameter(command, "@generated_at", snapshot.GeneratedAt);
|
||||
AddJsonbParameter(command, "@nodes_json", nodesJson);
|
||||
AddJsonbParameter(command, "@edges_json", edgesJson);
|
||||
AddParameter(command, "@queued_at", DateTimeOffset.UtcNow);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<GraphAnalyticsSnapshot>> GetPendingSnapshotsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT tenant, snapshot_id, generated_at, nodes_json, edges_json
|
||||
FROM graph.pending_snapshots
|
||||
ORDER BY queued_at ASC
|
||||
LIMIT 100";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<GraphAnalyticsSnapshot>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(MapSnapshot(reader));
|
||||
}
|
||||
|
||||
return results.ToImmutableArray();
|
||||
}
|
||||
|
||||
public async Task MarkProcessedAsync(string tenant, string snapshotId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(snapshotId);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
DELETE FROM graph.pending_snapshots
|
||||
WHERE tenant = @tenant AND snapshot_id = @snapshot_id";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@tenant", tenant ?? string.Empty);
|
||||
AddParameter(command, "@snapshot_id", snapshotId.Trim());
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static GraphAnalyticsSnapshot MapSnapshot(NpgsqlDataReader reader)
|
||||
{
|
||||
var tenant = reader.GetString(0);
|
||||
var snapshotId = reader.GetString(1);
|
||||
var generatedAt = reader.GetFieldValue<DateTimeOffset>(2);
|
||||
var nodesJson = reader.GetString(3);
|
||||
var edgesJson = reader.GetString(4);
|
||||
|
||||
var nodeStrings = JsonSerializer.Deserialize<List<string>>(nodesJson, JsonOptions) ?? new List<string>();
|
||||
var edgeStrings = JsonSerializer.Deserialize<List<string>>(edgesJson, JsonOptions) ?? new List<string>();
|
||||
|
||||
var nodes = nodeStrings
|
||||
.Select(s => JsonNode.Parse(s) as JsonObject)
|
||||
.Where(n => n is not null)
|
||||
.Cast<JsonObject>()
|
||||
.ToImmutableArray();
|
||||
|
||||
var edges = edgeStrings
|
||||
.Select(s => JsonNode.Parse(s) as JsonObject)
|
||||
.Where(e => e is not null)
|
||||
.Cast<JsonObject>()
|
||||
.ToImmutableArray();
|
||||
|
||||
return new GraphAnalyticsSnapshot(tenant, snapshotId, generatedAt, nodes, edges);
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS graph;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph.pending_snapshots (
|
||||
tenant TEXT NOT NULL,
|
||||
snapshot_id TEXT NOT NULL,
|
||||
generated_at TIMESTAMPTZ NOT NULL,
|
||||
nodes_json JSONB NOT NULL,
|
||||
edges_json JSONB NOT NULL,
|
||||
queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (tenant, snapshot_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pending_snapshots_queued_at ON graph.pending_snapshots (queued_at);";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Graph.Indexer.Incremental;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Graph.Indexer.Persistence.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IIdempotencyStore"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresIdempotencyStore : RepositoryBase<GraphIndexerDataSource>, IIdempotencyStore
|
||||
{
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresIdempotencyStore(GraphIndexerDataSource dataSource, ILogger<PostgresIdempotencyStore> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<bool> HasSeenAsync(string sequenceToken, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sequenceToken);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT EXISTS(SELECT 1 FROM graph.idempotency_tokens WHERE sequence_token = @sequence_token)";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@sequence_token", sequenceToken.Trim());
|
||||
|
||||
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
return result is bool seen && seen;
|
||||
}
|
||||
|
||||
public async Task MarkSeenAsync(string sequenceToken, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sequenceToken);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
INSERT INTO graph.idempotency_tokens (sequence_token, seen_at)
|
||||
VALUES (@sequence_token, @seen_at)
|
||||
ON CONFLICT (sequence_token) DO NOTHING";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@sequence_token", sequenceToken.Trim());
|
||||
AddParameter(command, "@seen_at", DateTimeOffset.UtcNow);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS graph;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph.idempotency_tokens (
|
||||
sequence_token TEXT PRIMARY KEY,
|
||||
seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_seen_at ON graph.idempotency_tokens (seen_at);";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<RootNamespace>StellaOps.Graph.Indexer.Persistence</RootNamespace>
|
||||
<AssemblyName>StellaOps.Graph.Indexer.Persistence</AssemblyName>
|
||||
<Description>Consolidated persistence layer for StellaOps Graph Indexer module</Description>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" PrivateAssets="all" />
|
||||
<PackageReference Include="Npgsql" />
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\StellaOps.Graph.Indexer\StellaOps.Graph.Indexer.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.Postgres\StellaOps.Infrastructure.Postgres.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.EfCore\StellaOps.Infrastructure.EfCore.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user