307 lines
12 KiB
C#
307 lines
12 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// © StellaOps Contributors. See LICENSE and NOTICE.md in the repository root.
|
|
|
|
using System.Diagnostics;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using StellaOps.VexLens.Consensus;
|
|
using StellaOps.VexLens.Models;
|
|
using StellaOps.VexLens.Options;
|
|
|
|
namespace StellaOps.VexLens.Storage;
|
|
|
|
/// <summary>
|
|
/// Dual-write implementation of <see cref="IConsensusProjectionStore"/> for migration.
|
|
/// Writes to both in-memory and PostgreSQL stores, reads from configurable primary.
|
|
/// Sprint: SPRINT_20251228_007_BE_sbom_lineage_graph_ii (LIN-BE-022)
|
|
/// </summary>
|
|
public sealed class DualWriteConsensusProjectionStore : IConsensusProjectionStore
|
|
{
|
|
private static readonly ActivitySource ActivitySource = new("StellaOps.VexLens.Storage.DualWriteConsensusProjectionStore");
|
|
|
|
private readonly IConsensusProjectionStore _memoryStore;
|
|
private readonly IConsensusProjectionStore _postgresStore;
|
|
private readonly VexLensStorageOptions _options;
|
|
private readonly ILogger<DualWriteConsensusProjectionStore> _logger;
|
|
private readonly bool _readFromPostgres;
|
|
private readonly bool _logDiscrepancies;
|
|
|
|
/// <summary>
|
|
/// Creates a dual-write store for migration from in-memory to PostgreSQL.
|
|
/// </summary>
|
|
/// <param name="memoryStore">The in-memory store (primary during migration).</param>
|
|
/// <param name="postgresStore">The PostgreSQL store (target for migration).</param>
|
|
/// <param name="options">Storage options including read preference.</param>
|
|
/// <param name="logger">Logger for discrepancy reporting.</param>
|
|
public DualWriteConsensusProjectionStore(
|
|
IConsensusProjectionStore memoryStore,
|
|
IConsensusProjectionStore postgresStore,
|
|
IOptions<VexLensStorageOptions> options,
|
|
ILogger<DualWriteConsensusProjectionStore> logger)
|
|
{
|
|
_memoryStore = memoryStore ?? throw new ArgumentNullException(nameof(memoryStore));
|
|
_postgresStore = postgresStore ?? throw new ArgumentNullException(nameof(postgresStore));
|
|
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
|
|
_readFromPostgres = string.Equals(_options.DualWriteReadFrom, "postgres", StringComparison.OrdinalIgnoreCase);
|
|
_logDiscrepancies = _options.LogDualWriteDiscrepancies;
|
|
|
|
_logger.LogInformation(
|
|
"DualWriteConsensusProjectionStore initialized. ReadFrom={ReadFrom}, LogDiscrepancies={LogDiscrepancies}",
|
|
_options.DualWriteReadFrom, _logDiscrepancies);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the primary store for reads based on configuration.
|
|
/// </summary>
|
|
private IConsensusProjectionStore PrimaryStore => _readFromPostgres ? _postgresStore : _memoryStore;
|
|
|
|
/// <summary>
|
|
/// Gets the secondary store for writes (opposite of primary).
|
|
/// </summary>
|
|
private IConsensusProjectionStore SecondaryStore => _readFromPostgres ? _memoryStore : _postgresStore;
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ConsensusProjection> StoreAsync(
|
|
VexConsensusResult result,
|
|
StoreProjectionOptions options,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
using var activity = ActivitySource.StartActivity("StoreAsync.DualWrite");
|
|
activity?.SetTag("vulnerabilityId", result.VulnerabilityId);
|
|
activity?.SetTag("productKey", result.ProductKey);
|
|
|
|
// Write to primary first
|
|
var primaryResult = await PrimaryStore.StoreAsync(result, options, cancellationToken);
|
|
activity?.SetTag("primaryProjectionId", primaryResult.ProjectionId);
|
|
|
|
// Write to secondary (fire-and-forget with error logging)
|
|
try
|
|
{
|
|
var secondaryResult = await SecondaryStore.StoreAsync(result, options, cancellationToken);
|
|
activity?.SetTag("secondaryProjectionId", secondaryResult.ProjectionId);
|
|
|
|
if (_logDiscrepancies)
|
|
{
|
|
await ValidateStoreResultsAsync(primaryResult, secondaryResult, activity);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(
|
|
ex,
|
|
"Dual-write to secondary store failed for {VulnerabilityId}/{ProductKey}. Primary write succeeded.",
|
|
result.VulnerabilityId, result.ProductKey);
|
|
activity?.SetTag("secondaryWriteFailed", true);
|
|
}
|
|
|
|
return primaryResult;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ConsensusProjection?> GetAsync(
|
|
string projectionId,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
using var activity = ActivitySource.StartActivity("GetAsync.DualWrite");
|
|
activity?.SetTag("projectionId", projectionId);
|
|
activity?.SetTag("readFrom", _readFromPostgres ? "postgres" : "memory");
|
|
|
|
var result = await PrimaryStore.GetAsync(projectionId, cancellationToken);
|
|
|
|
if (_logDiscrepancies && result is not null)
|
|
{
|
|
_ = Task.Run(async () =>
|
|
{
|
|
try
|
|
{
|
|
var secondaryResult = await SecondaryStore.GetAsync(projectionId, CancellationToken.None);
|
|
if (secondaryResult is null)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy: Projection {ProjectionId} found in primary but not in secondary",
|
|
projectionId);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Secondary lookup failed during discrepancy check for {ProjectionId}", projectionId);
|
|
}
|
|
}, cancellationToken);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ConsensusProjection?> GetLatestAsync(
|
|
string vulnerabilityId,
|
|
string productKey,
|
|
string? tenantId = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
using var activity = ActivitySource.StartActivity("GetLatestAsync.DualWrite");
|
|
activity?.SetTag("vulnerabilityId", vulnerabilityId);
|
|
activity?.SetTag("productKey", productKey);
|
|
activity?.SetTag("readFrom", _readFromPostgres ? "postgres" : "memory");
|
|
|
|
var result = await PrimaryStore.GetLatestAsync(vulnerabilityId, productKey, tenantId, cancellationToken);
|
|
|
|
if (_logDiscrepancies)
|
|
{
|
|
_ = Task.Run(async () =>
|
|
{
|
|
try
|
|
{
|
|
var secondaryResult = await SecondaryStore.GetLatestAsync(
|
|
vulnerabilityId, productKey, tenantId, CancellationToken.None);
|
|
|
|
ValidateProjectionConsistency(result, secondaryResult, vulnerabilityId, productKey);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(
|
|
ex,
|
|
"Secondary lookup failed during discrepancy check for {VulnerabilityId}/{ProductKey}",
|
|
vulnerabilityId, productKey);
|
|
}
|
|
}, cancellationToken);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ProjectionListResult> ListAsync(
|
|
ProjectionQuery query,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
using var activity = ActivitySource.StartActivity("ListAsync.DualWrite");
|
|
activity?.SetTag("readFrom", _readFromPostgres ? "postgres" : "memory");
|
|
|
|
// List operations only read from primary for consistency
|
|
return await PrimaryStore.ListAsync(query, cancellationToken);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IReadOnlyList<ConsensusProjection>> GetHistoryAsync(
|
|
string vulnerabilityId,
|
|
string productKey,
|
|
string? tenantId = null,
|
|
int? limit = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
using var activity = ActivitySource.StartActivity("GetHistoryAsync.DualWrite");
|
|
activity?.SetTag("vulnerabilityId", vulnerabilityId);
|
|
activity?.SetTag("productKey", productKey);
|
|
activity?.SetTag("readFrom", _readFromPostgres ? "postgres" : "memory");
|
|
|
|
return await PrimaryStore.GetHistoryAsync(vulnerabilityId, productKey, tenantId, limit, cancellationToken);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<int> PurgeAsync(
|
|
DateTimeOffset olderThan,
|
|
string? tenantId = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
using var activity = ActivitySource.StartActivity("PurgeAsync.DualWrite");
|
|
activity?.SetTag("olderThan", olderThan.ToString("O"));
|
|
|
|
// Purge from both stores
|
|
var primaryCount = await PrimaryStore.PurgeAsync(olderThan, tenantId, cancellationToken);
|
|
activity?.SetTag("primaryPurgedCount", primaryCount);
|
|
|
|
try
|
|
{
|
|
var secondaryCount = await SecondaryStore.PurgeAsync(olderThan, tenantId, cancellationToken);
|
|
activity?.SetTag("secondaryPurgedCount", secondaryCount);
|
|
|
|
if (_logDiscrepancies && primaryCount != secondaryCount)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy: Purge count differs. Primary={PrimaryCount}, Secondary={SecondaryCount}",
|
|
primaryCount, secondaryCount);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Dual-write purge failed for secondary store");
|
|
}
|
|
|
|
return primaryCount;
|
|
}
|
|
|
|
private async Task ValidateStoreResultsAsync(
|
|
ConsensusProjection primary,
|
|
ConsensusProjection secondary,
|
|
Activity? activity)
|
|
{
|
|
// Basic validation - the key fields should match
|
|
if (primary.VulnerabilityId != secondary.VulnerabilityId ||
|
|
primary.ProductKey != secondary.ProductKey ||
|
|
primary.Status != secondary.Status ||
|
|
Math.Abs(primary.ConfidenceScore - secondary.ConfidenceScore) > 0.001)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy detected for {VulnerabilityId}/{ProductKey}. " +
|
|
"Primary: Status={PrimaryStatus}, Confidence={PrimaryConfidence}. " +
|
|
"Secondary: Status={SecondaryStatus}, Confidence={SecondaryConfidence}",
|
|
primary.VulnerabilityId,
|
|
primary.ProductKey,
|
|
primary.Status,
|
|
primary.ConfidenceScore,
|
|
secondary.Status,
|
|
secondary.ConfidenceScore);
|
|
|
|
activity?.SetTag("discrepancyDetected", true);
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
}
|
|
|
|
private void ValidateProjectionConsistency(
|
|
ConsensusProjection? primary,
|
|
ConsensusProjection? secondary,
|
|
string vulnerabilityId,
|
|
string productKey)
|
|
{
|
|
if (primary is null && secondary is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (primary is null && secondary is not null)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy: Projection for {VulnerabilityId}/{ProductKey} exists in secondary but not primary",
|
|
vulnerabilityId, productKey);
|
|
return;
|
|
}
|
|
|
|
if (primary is not null && secondary is null)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy: Projection for {VulnerabilityId}/{ProductKey} exists in primary but not secondary",
|
|
vulnerabilityId, productKey);
|
|
return;
|
|
}
|
|
|
|
// Both exist - compare key fields
|
|
if (primary!.Status != secondary!.Status)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy: Status mismatch for {VulnerabilityId}/{ProductKey}. Primary={PrimaryStatus}, Secondary={SecondaryStatus}",
|
|
vulnerabilityId, productKey, primary.Status, secondary.Status);
|
|
}
|
|
|
|
if (Math.Abs(primary.ConfidenceScore - secondary.ConfidenceScore) > 0.001)
|
|
{
|
|
_logger.LogWarning(
|
|
"Dual-write discrepancy: Confidence mismatch for {VulnerabilityId}/{ProductKey}. Primary={PrimaryConfidence}, Secondary={SecondaryConfidence}",
|
|
vulnerabilityId, productKey, primary.ConfidenceScore, secondary.ConfidenceScore);
|
|
}
|
|
}
|
|
}
|