audit work, fixed StellaOps.sln warnings/errors, fixed tests, sprints work, new advisories

This commit is contained in:
master
2026-01-07 18:49:59 +02:00
parent 04ec098046
commit 608a7f85c0
866 changed files with 56323 additions and 6231 deletions

View File

@@ -0,0 +1,129 @@
// -----------------------------------------------------------------------------
// CveObservationNode.cs
// Sprint: SPRINT_20260106_001_004_BE_determinization_integration
// Task: DBI-008 - Create CveObservationNode record in Graph
// Description: Represents a CVE observation in the graph with signal state
// -----------------------------------------------------------------------------
using System.Text.Json.Serialization;
namespace StellaOps.Graph.Core;
/// <summary>
/// Represents a CVE observation node in the graph.
/// </summary>
public sealed record CveObservationNode
{
/// <summary>Content-addressed node ID (sha256 of canonical form).</summary>
[JsonPropertyName("node_id")]
public required string NodeId { get; init; }
/// <summary>The CVE ID.</summary>
[JsonPropertyName("cve_id")]
public required string CveId { get; init; }
/// <summary>The product (purl or cpe).</summary>
[JsonPropertyName("product")]
public required string Product { get; init; }
/// <summary>Tenant ID for multi-tenancy.</summary>
[JsonPropertyName("tenant_id")]
public required string TenantId { get; init; }
/// <summary>Current observation state.</summary>
[JsonPropertyName("state")]
public required ObservationState State { get; init; }
/// <summary>Uncertainty score (0.0 = certain, 1.0 = highly uncertain).</summary>
[JsonPropertyName("uncertainty_score")]
public double UncertaintyScore { get; init; }
/// <summary>EPSS signal state.</summary>
[JsonPropertyName("epss")]
public SignalSnapshot? Epss { get; init; }
/// <summary>KEV signal state.</summary>
[JsonPropertyName("kev")]
public SignalSnapshot? Kev { get; init; }
/// <summary>VEX signal state.</summary>
[JsonPropertyName("vex")]
public SignalSnapshot? Vex { get; init; }
/// <summary>Reachability signal state.</summary>
[JsonPropertyName("reachability")]
public SignalSnapshot? Reachability { get; init; }
/// <summary>Missing signals that couldn't be fetched.</summary>
[JsonPropertyName("missing_signals")]
public IReadOnlyList<string>? MissingSignals { get; init; }
/// <summary>When this node was created.</summary>
[JsonPropertyName("created_at")]
public required DateTimeOffset CreatedAt { get; init; }
/// <summary>When this node was last updated.</summary>
[JsonPropertyName("updated_at")]
public required DateTimeOffset UpdatedAt { get; init; }
/// <summary>Schema version for forward compatibility.</summary>
[JsonPropertyName("schema_version")]
public string SchemaVersion { get; init; } = "1.0";
}
/// <summary>
/// Observation state for a CVE.
/// </summary>
[JsonConverter(typeof(JsonStringEnumConverter))]
public enum ObservationState
{
/// <summary>Initial state, not yet processed.</summary>
Pending,
/// <summary>Actively being investigated.</summary>
Investigating,
/// <summary>Confirmed affected.</summary>
Affected,
/// <summary>Confirmed not affected.</summary>
NotAffected,
/// <summary>Fixed in a later version.</summary>
Fixed,
/// <summary>Mitigated by other means.</summary>
Mitigated,
/// <summary>Accepted risk.</summary>
Accepted,
/// <summary>False positive.</summary>
FalsePositive
}
/// <summary>
/// Snapshot of a signal at a point in time.
/// </summary>
public sealed record SignalSnapshot
{
/// <summary>Signal status.</summary>
[JsonPropertyName("status")]
public required string Status { get; init; }
/// <summary>Signal value summary (JSON-serializable).</summary>
[JsonPropertyName("value_summary")]
public string? ValueSummary { get; init; }
/// <summary>When the signal was captured.</summary>
[JsonPropertyName("captured_at")]
public required DateTimeOffset CapturedAt { get; init; }
/// <summary>Source of the signal.</summary>
[JsonPropertyName("source")]
public string? Source { get; init; }
/// <summary>Time-to-live remaining.</summary>
[JsonPropertyName("ttl_remaining")]
public TimeSpan? TtlRemaining { get; init; }
}

View File

@@ -0,0 +1,128 @@
// -----------------------------------------------------------------------------
// ICveObservationNodeRepository.cs
// Sprint: SPRINT_20260106_001_004_BE_determinization_integration
// Task: DBI-009 - Create ICveObservationNodeRepository interface
// Description: Repository interface for CVE observation nodes
// -----------------------------------------------------------------------------
namespace StellaOps.Graph.Core;
/// <summary>
/// Repository for CVE observation nodes.
/// </summary>
public interface ICveObservationNodeRepository
{
/// <summary>
/// Creates or updates an observation node.
/// </summary>
Task<CveObservationNode> UpsertAsync(
CveObservationNode node,
CancellationToken ct = default);
/// <summary>
/// Gets an observation node by ID.
/// </summary>
Task<CveObservationNode?> GetByIdAsync(
string nodeId,
CancellationToken ct = default);
/// <summary>
/// Gets observation nodes for a CVE.
/// </summary>
Task<IReadOnlyList<CveObservationNode>> GetByCveAsync(
string cveId,
string tenantId,
CancellationToken ct = default);
/// <summary>
/// Gets observation nodes for a product.
/// </summary>
Task<IReadOnlyList<CveObservationNode>> GetByProductAsync(
string product,
string tenantId,
CancellationToken ct = default);
/// <summary>
/// Gets observation nodes by state.
/// </summary>
Task<IReadOnlyList<CveObservationNode>> GetByStateAsync(
ObservationState state,
string tenantId,
int limit = 100,
int offset = 0,
CancellationToken ct = default);
/// <summary>
/// Gets nodes with high uncertainty scores.
/// </summary>
Task<IReadOnlyList<CveObservationNode>> GetHighUncertaintyAsync(
double threshold,
string tenantId,
int limit = 100,
CancellationToken ct = default);
/// <summary>
/// Gets nodes with missing signals.
/// </summary>
Task<IReadOnlyList<CveObservationNode>> GetWithMissingSignalsAsync(
string signalType,
string tenantId,
int limit = 100,
CancellationToken ct = default);
/// <summary>
/// Gets nodes updated since a timestamp.
/// </summary>
Task<IReadOnlyList<CveObservationNode>> GetUpdatedSinceAsync(
DateTimeOffset since,
string tenantId,
int limit = 1000,
CancellationToken ct = default);
/// <summary>
/// Deletes an observation node.
/// </summary>
Task<bool> DeleteAsync(
string nodeId,
CancellationToken ct = default);
/// <summary>
/// Counts observation nodes by state.
/// </summary>
Task<IDictionary<ObservationState, int>> CountByStateAsync(
string tenantId,
CancellationToken ct = default);
}
/// <summary>
/// Options for querying observation nodes.
/// </summary>
public sealed record CveObservationQueryOptions
{
/// <summary>Filter by CVE ID.</summary>
public string? CveId { get; init; }
/// <summary>Filter by product.</summary>
public string? Product { get; init; }
/// <summary>Filter by state.</summary>
public ObservationState? State { get; init; }
/// <summary>Minimum uncertainty score.</summary>
public double? MinUncertainty { get; init; }
/// <summary>Maximum uncertainty score.</summary>
public double? MaxUncertainty { get; init; }
/// <summary>Filter by missing signal.</summary>
public string? MissingSignal { get; init; }
/// <summary>Updated since timestamp.</summary>
public DateTimeOffset? UpdatedSince { get; init; }
/// <summary>Maximum results.</summary>
public int Limit { get; init; } = 100;
/// <summary>Offset for pagination.</summary>
public int Offset { get; init; }
}

View File

@@ -0,0 +1,347 @@
// -----------------------------------------------------------------------------
// PostgresCveObservationNodeRepository.cs
// Sprint: SPRINT_20260106_001_004_BE_determinization_integration
// Task: DBI-010 - Implement PostgresCveObservationNodeRepository
// Description: PostgreSQL implementation of CVE observation node repository
// -----------------------------------------------------------------------------
using System.Globalization;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
namespace StellaOps.Graph.Core;
/// <summary>
/// PostgreSQL implementation of CVE observation node repository.
/// </summary>
public sealed class PostgresCveObservationNodeRepository : ICveObservationNodeRepository
{
private readonly NpgsqlDataSource _dataSource;
private readonly TimeProvider _timeProvider;
private readonly ILogger<PostgresCveObservationNodeRepository> _logger;
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
WriteIndented = false
};
public PostgresCveObservationNodeRepository(
NpgsqlDataSource dataSource,
TimeProvider timeProvider,
ILogger<PostgresCveObservationNodeRepository> logger)
{
_dataSource = dataSource;
_timeProvider = timeProvider;
_logger = logger;
}
/// <inheritdoc />
public async Task<CveObservationNode> UpsertAsync(
CveObservationNode node,
CancellationToken ct = default)
{
const string sql = """
INSERT INTO cve_observation_nodes (
node_id, cve_id, product, tenant_id, state, uncertainty_score,
epss, kev, vex, reachability, missing_signals,
created_at, updated_at, schema_version
) VALUES (
@node_id, @cve_id, @product, @tenant_id, @state, @uncertainty_score,
@epss, @kev, @vex, @reachability, @missing_signals,
@created_at, @updated_at, @schema_version
)
ON CONFLICT (node_id) DO UPDATE SET
state = EXCLUDED.state,
uncertainty_score = EXCLUDED.uncertainty_score,
epss = EXCLUDED.epss,
kev = EXCLUDED.kev,
vex = EXCLUDED.vex,
reachability = EXCLUDED.reachability,
missing_signals = EXCLUDED.missing_signals,
updated_at = EXCLUDED.updated_at
RETURNING *
""";
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
cmd.Parameters.AddWithValue("node_id", node.NodeId);
cmd.Parameters.AddWithValue("cve_id", node.CveId);
cmd.Parameters.AddWithValue("product", node.Product);
cmd.Parameters.AddWithValue("tenant_id", node.TenantId);
cmd.Parameters.AddWithValue("state", node.State.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue("uncertainty_score", node.UncertaintyScore);
cmd.Parameters.AddWithValue("epss", SerializeSignal(node.Epss));
cmd.Parameters.AddWithValue("kev", SerializeSignal(node.Kev));
cmd.Parameters.AddWithValue("vex", SerializeSignal(node.Vex));
cmd.Parameters.AddWithValue("reachability", SerializeSignal(node.Reachability));
cmd.Parameters.AddWithValue("missing_signals", SerializeList(node.MissingSignals));
cmd.Parameters.AddWithValue("created_at", node.CreatedAt);
cmd.Parameters.AddWithValue("updated_at", node.UpdatedAt);
cmd.Parameters.AddWithValue("schema_version", node.SchemaVersion);
await using var reader = await cmd.ExecuteReaderAsync(ct);
if (await reader.ReadAsync(ct))
{
return MapFromReader(reader);
}
throw new InvalidOperationException("Upsert did not return a row");
}
/// <inheritdoc />
public async Task<CveObservationNode?> GetByIdAsync(
string nodeId,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes WHERE node_id = @node_id
""";
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
cmd.Parameters.AddWithValue("node_id", nodeId);
await using var reader = await cmd.ExecuteReaderAsync(ct);
if (await reader.ReadAsync(ct))
{
return MapFromReader(reader);
}
return null;
}
/// <inheritdoc />
public async Task<IReadOnlyList<CveObservationNode>> GetByCveAsync(
string cveId,
string tenantId,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes
WHERE cve_id = @cve_id AND tenant_id = @tenant_id
ORDER BY updated_at DESC
""";
return await ExecuteQueryAsync(sql, new { cve_id = cveId, tenant_id = tenantId }, ct);
}
/// <inheritdoc />
public async Task<IReadOnlyList<CveObservationNode>> GetByProductAsync(
string product,
string tenantId,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes
WHERE product = @product AND tenant_id = @tenant_id
ORDER BY updated_at DESC
""";
return await ExecuteQueryAsync(sql, new { product, tenant_id = tenantId }, ct);
}
/// <inheritdoc />
public async Task<IReadOnlyList<CveObservationNode>> GetByStateAsync(
ObservationState state,
string tenantId,
int limit = 100,
int offset = 0,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes
WHERE state = @state AND tenant_id = @tenant_id
ORDER BY updated_at DESC
LIMIT @limit OFFSET @offset
""";
return await ExecuteQueryAsync(sql, new
{
state = state.ToString().ToLowerInvariant(),
tenant_id = tenantId,
limit,
offset
}, ct);
}
/// <inheritdoc />
public async Task<IReadOnlyList<CveObservationNode>> GetHighUncertaintyAsync(
double threshold,
string tenantId,
int limit = 100,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes
WHERE uncertainty_score >= @threshold AND tenant_id = @tenant_id
ORDER BY uncertainty_score DESC
LIMIT @limit
""";
return await ExecuteQueryAsync(sql, new { threshold, tenant_id = tenantId, limit }, ct);
}
/// <inheritdoc />
public async Task<IReadOnlyList<CveObservationNode>> GetWithMissingSignalsAsync(
string signalType,
string tenantId,
int limit = 100,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes
WHERE missing_signals ? @signal_type AND tenant_id = @tenant_id
ORDER BY updated_at DESC
LIMIT @limit
""";
return await ExecuteQueryAsync(sql, new { signal_type = signalType, tenant_id = tenantId, limit }, ct);
}
/// <inheritdoc />
public async Task<IReadOnlyList<CveObservationNode>> GetUpdatedSinceAsync(
DateTimeOffset since,
string tenantId,
int limit = 1000,
CancellationToken ct = default)
{
const string sql = """
SELECT * FROM cve_observation_nodes
WHERE updated_at > @since AND tenant_id = @tenant_id
ORDER BY updated_at ASC
LIMIT @limit
""";
return await ExecuteQueryAsync(sql, new { since, tenant_id = tenantId, limit }, ct);
}
/// <inheritdoc />
public async Task<bool> DeleteAsync(
string nodeId,
CancellationToken ct = default)
{
const string sql = """
DELETE FROM cve_observation_nodes WHERE node_id = @node_id
""";
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
cmd.Parameters.AddWithValue("node_id", nodeId);
var affected = await cmd.ExecuteNonQueryAsync(ct);
return affected > 0;
}
/// <inheritdoc />
public async Task<IDictionary<ObservationState, int>> CountByStateAsync(
string tenantId,
CancellationToken ct = default)
{
const string sql = """
SELECT state, COUNT(*) as count
FROM cve_observation_nodes
WHERE tenant_id = @tenant_id
GROUP BY state
""";
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
cmd.Parameters.AddWithValue("tenant_id", tenantId);
var result = new Dictionary<ObservationState, int>();
await using var reader = await cmd.ExecuteReaderAsync(ct);
while (await reader.ReadAsync(ct))
{
var stateStr = reader.GetString(0);
var count = reader.GetInt32(1);
if (Enum.TryParse<ObservationState>(stateStr, ignoreCase: true, out var state))
{
result[state] = count;
}
}
return result;
}
private async Task<IReadOnlyList<CveObservationNode>> ExecuteQueryAsync(
string sql,
object parameters,
CancellationToken ct)
{
var results = new List<CveObservationNode>();
await using var conn = await _dataSource.OpenConnectionAsync(ct);
await using var cmd = new NpgsqlCommand(sql, conn);
// Add parameters from anonymous object
foreach (var prop in parameters.GetType().GetProperties())
{
var value = prop.GetValue(parameters);
cmd.Parameters.AddWithValue(prop.Name, value ?? DBNull.Value);
}
await using var reader = await cmd.ExecuteReaderAsync(ct);
while (await reader.ReadAsync(ct))
{
results.Add(MapFromReader(reader));
}
return results;
}
private static CveObservationNode MapFromReader(NpgsqlDataReader reader)
{
return new CveObservationNode
{
NodeId = reader.GetString(reader.GetOrdinal("node_id")),
CveId = reader.GetString(reader.GetOrdinal("cve_id")),
Product = reader.GetString(reader.GetOrdinal("product")),
TenantId = reader.GetString(reader.GetOrdinal("tenant_id")),
State = Enum.Parse<ObservationState>(
reader.GetString(reader.GetOrdinal("state")),
ignoreCase: true),
UncertaintyScore = reader.GetDouble(reader.GetOrdinal("uncertainty_score")),
Epss = DeserializeSignal(reader, "epss"),
Kev = DeserializeSignal(reader, "kev"),
Vex = DeserializeSignal(reader, "vex"),
Reachability = DeserializeSignal(reader, "reachability"),
MissingSignals = DeserializeList(reader, "missing_signals"),
CreatedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("created_at")),
UpdatedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("updated_at")),
SchemaVersion = reader.GetString(reader.GetOrdinal("schema_version"))
};
}
private static object SerializeSignal(SignalSnapshot? signal)
{
if (signal is null) return DBNull.Value;
return JsonSerializer.Serialize(signal, JsonOptions);
}
private static object SerializeList(IReadOnlyList<string>? list)
{
if (list is null || list.Count == 0) return DBNull.Value;
return JsonSerializer.Serialize(list, JsonOptions);
}
private static SignalSnapshot? DeserializeSignal(NpgsqlDataReader reader, string column)
{
var ordinal = reader.GetOrdinal(column);
if (reader.IsDBNull(ordinal)) return null;
var json = reader.GetString(ordinal);
return JsonSerializer.Deserialize<SignalSnapshot>(json, JsonOptions);
}
private static IReadOnlyList<string>? DeserializeList(NpgsqlDataReader reader, string column)
{
var ordinal = reader.GetOrdinal(column);
if (reader.IsDBNull(ordinal)) return null;
var json = reader.GetString(ordinal);
return JsonSerializer.Deserialize<List<string>>(json, JsonOptions);
}
}

View File

@@ -0,0 +1,121 @@
-- -----------------------------------------------------------------------------
-- 003_cve_observation_nodes.sql
-- Sprint: SPRINT_20260106_001_004_BE_determinization_integration
-- Task: DBI-011 - Create migration for cve_observation_nodes table
-- Description: Creates the cve_observation_nodes table for graph persistence
-- -----------------------------------------------------------------------------
-- CVE Observation Nodes table
-- Stores CVE observations with signal state for the determinization pipeline
CREATE TABLE IF NOT EXISTS cve_observation_nodes (
-- Primary key: content-addressed hash ID
node_id TEXT PRIMARY KEY,
-- Core identifiers
cve_id TEXT NOT NULL,
product TEXT NOT NULL,
tenant_id TEXT NOT NULL,
-- Observation state
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'investigating', 'affected', 'not_affected',
'fixed', 'mitigated', 'accepted', 'false_positive')),
-- Uncertainty score (0.0 = certain, 1.0 = highly uncertain)
uncertainty_score DOUBLE PRECISION NOT NULL DEFAULT 1.0
CHECK (uncertainty_score >= 0.0 AND uncertainty_score <= 1.0),
-- Signal snapshots (JSONB for flexible schema)
epss JSONB,
kev JSONB,
vex JSONB,
reachability JSONB,
-- Missing signals that couldn't be fetched
missing_signals JSONB DEFAULT '[]'::JSONB,
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Schema version for forward compatibility
schema_version TEXT NOT NULL DEFAULT '1.0',
-- Ensure unique CVE+product+tenant combination
CONSTRAINT uq_cve_product_tenant UNIQUE (cve_id, product, tenant_id)
);
-- Indexes for common query patterns
-- Query by CVE (most common)
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_cve_id
ON cve_observation_nodes (cve_id, tenant_id);
-- Query by product
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_product
ON cve_observation_nodes (product, tenant_id);
-- Query by state (for dashboards, reports)
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_state
ON cve_observation_nodes (state, tenant_id);
-- Query by uncertainty (for triage prioritization)
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_uncertainty
ON cve_observation_nodes (uncertainty_score DESC, tenant_id)
WHERE uncertainty_score > 0.5;
-- Query by updated_at (for delta sync)
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_updated
ON cve_observation_nodes (updated_at DESC, tenant_id);
-- Query for missing signals (to retry failed lookups)
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_missing_signals
ON cve_observation_nodes USING GIN (missing_signals);
-- Query KEV status from signal (for KEV-related queries)
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_kev
ON cve_observation_nodes ((kev->>'status'), tenant_id)
WHERE kev IS NOT NULL;
-- Query VEX status from signal
CREATE INDEX IF NOT EXISTS idx_cve_observation_nodes_vex
ON cve_observation_nodes ((vex->>'status'), tenant_id)
WHERE vex IS NOT NULL;
-- Trigger for updated_at
CREATE OR REPLACE FUNCTION update_cve_observation_nodes_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_cve_observation_nodes_updated_at
ON cve_observation_nodes;
CREATE TRIGGER trg_cve_observation_nodes_updated_at
BEFORE UPDATE ON cve_observation_nodes
FOR EACH ROW
EXECUTE FUNCTION update_cve_observation_nodes_updated_at();
-- Row-level security for multi-tenancy
ALTER TABLE cve_observation_nodes ENABLE ROW LEVEL SECURITY;
-- Policy: users can only access their own tenant's data
CREATE POLICY tenant_isolation ON cve_observation_nodes
USING (tenant_id = current_setting('app.tenant_id', true))
WITH CHECK (tenant_id = current_setting('app.tenant_id', true));
-- Grant permissions (adjust role name as needed)
GRANT SELECT, INSERT, UPDATE, DELETE ON cve_observation_nodes TO stellaops_app;
-- Comments
COMMENT ON TABLE cve_observation_nodes IS
'CVE observation nodes with signal state for the determinization pipeline';
COMMENT ON COLUMN cve_observation_nodes.node_id IS
'Content-addressed hash ID (SHA-256 of canonical form)';
COMMENT ON COLUMN cve_observation_nodes.uncertainty_score IS
'Uncertainty score: 0.0 = certain, 1.0 = highly uncertain';
COMMENT ON COLUMN cve_observation_nodes.missing_signals IS
'Array of signal types that failed to fetch';