Refactor and update test projects, remove obsolete tests, and upgrade dependencies

- Deleted obsolete test files for SchedulerAuditService and SchedulerMongoSessionFactory.
- Removed unused TestDataFactory class.
- Updated project files for Mongo.Tests to remove references to deleted files.
- Upgraded BouncyCastle.Cryptography package to version 2.6.2 across multiple projects.
- Replaced Microsoft.Extensions.Http.Polly with Microsoft.Extensions.Http.Resilience in Zastava.Webhook project.
- Updated NetEscapades.Configuration.Yaml package to version 3.1.0 in Configuration library.
- Upgraded Pkcs11Interop package to version 5.1.2 in Cryptography libraries.
- Refactored Argon2idPasswordHasher to use BouncyCastle for hashing instead of Konscious.
- Updated JsonSchema.Net package to version 7.3.2 in Microservice project.
- Updated global.json to use .NET SDK version 10.0.101.
This commit is contained in:
master
2025-12-10 19:13:29 +02:00
parent a3c7fe5e88
commit b7059d523e
369 changed files with 11125 additions and 14245 deletions

View File

@@ -0,0 +1,90 @@
using System.Collections.Immutable;
namespace StellaOps.Excititor.Core.Storage;
public sealed class DuplicateAirgapImportException : Exception
{
public DuplicateAirgapImportException(string message)
: base(message)
{
}
}
/// <summary>
/// Timeline entry for an imported airgap bundle.
/// </summary>
public sealed record AirgapTimelineEntry
{
public string EventType { get; init; } = string.Empty;
public DateTimeOffset CreatedAt { get; init; }
public string TenantId { get; init; } = "default";
public string BundleId { get; init; } = string.Empty;
public string MirrorGeneration { get; init; } = string.Empty;
public int? StalenessSeconds { get; init; }
public string? ErrorCode { get; init; }
public string? Message { get; init; }
public string? Remediation { get; init; }
public string? Actor { get; init; }
public string? Scopes { get; init; }
}
/// <summary>
/// Persisted airgap import record describing a mirror bundle and associated metadata.
/// </summary>
public sealed record AirgapImportRecord
{
public string Id { get; init; } = string.Empty;
public string TenantId { get; init; } = "default";
public string BundleId { get; init; } = string.Empty;
public string MirrorGeneration { get; init; } = "0";
public string Publisher { get; init; } = string.Empty;
public DateTimeOffset SignedAt { get; init; }
public DateTimeOffset ImportedAt { get; init; }
public string PayloadHash { get; init; } = string.Empty;
public string? PayloadUrl { get; init; }
public string Signature { get; init; } = string.Empty;
public string? TransparencyLog { get; init; }
public string? PortableManifestPath { get; init; }
public string? PortableManifestHash { get; init; }
public string? EvidenceLockerPath { get; init; }
public IReadOnlyList<AirgapTimelineEntry> Timeline { get; init; } = Array.Empty<AirgapTimelineEntry>();
public string? ImportActor { get; init; }
public string? ImportScopes { get; init; }
}
public interface IAirgapImportStore
{
Task SaveAsync(AirgapImportRecord record, CancellationToken cancellationToken);
Task<AirgapImportRecord?> FindByBundleIdAsync(string tenantId, string bundleId, string? mirrorGeneration, CancellationToken cancellationToken);
Task<IReadOnlyList<AirgapImportRecord>> ListAsync(string tenantId, string? publisherFilter, DateTimeOffset? importedAfter, int limit, int offset, CancellationToken cancellationToken);
Task<int> CountAsync(string tenantId, string? publisherFilter, DateTimeOffset? importedAfter, CancellationToken cancellationToken);
}

View File

@@ -11,16 +11,24 @@ public sealed record VexConnectorState(
string ConnectorId,
DateTimeOffset? LastUpdated,
ImmutableArray<string> DocumentDigests,
ImmutableDictionary<string, string> ResumeTokens = default,
ImmutableDictionary<string, string>? ResumeTokens = null,
DateTimeOffset? LastSuccessAt = null,
int FailureCount = 0,
DateTimeOffset? NextEligibleRun = null,
string? LastFailureReason = null,
DateTimeOffset? LastCheckpoint = null)
DateTimeOffset? LastCheckpoint = null,
DateTimeOffset? LastHeartbeatAt = null,
string? LastHeartbeatStatus = null,
string? LastArtifactHash = null,
string? LastArtifactKind = null)
{
public ImmutableDictionary<string, string> ResumeTokens { get; init; } = ResumeTokens.IsDefault
? ImmutableDictionary<string, string>.Empty
: ResumeTokens;
public ImmutableArray<string> DocumentDigests { get; init; } =
DocumentDigests.IsDefault ? ImmutableArray<string>.Empty : DocumentDigests;
public ImmutableDictionary<string, string> ResumeTokens { get; init; } =
ResumeTokens is null || ResumeTokens.Count == 0
? ImmutableDictionary<string, string>.Empty
: ResumeTokens;
};
/// <summary>

View File

@@ -212,7 +212,7 @@ public sealed class InMemoryVexRawStore : IVexRawStore
private static byte[] CanonicalizeJson(ReadOnlyMemory<byte> content)
{
using var jsonDocument = JsonDocument.Parse(content);
using var buffer = new ArrayBufferWriter<byte>();
var buffer = new ArrayBufferWriter<byte>();
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false }))
{
WriteCanonical(writer, jsonDocument.RootElement);
@@ -396,7 +396,7 @@ public sealed class InMemoryAppendOnlyLinksetStore : IAppendOnlyLinksetStore, IV
tenant,
vulnerabilityId,
productKey,
new VexProductScope(productKey, null, null, productKey, null, Array.Empty<string>()),
new VexProductScope(productKey, "unknown", null, productKey, null, ImmutableArray<string>.Empty),
Enumerable.Empty<VexLinksetObservationRefModel>(),
Enumerable.Empty<VexObservationDisagreement>(),
DateTimeOffset.UtcNow,
@@ -554,7 +554,7 @@ public sealed class InMemoryAppendOnlyLinksetStore : IAppendOnlyLinksetStore, IV
return ValueTask.FromResult(existing);
}
var scope = new VexProductScope(productKey, null, null, productKey, null, Array.Empty<string>());
var scope = new VexProductScope(productKey, "unknown", null, productKey, null, ImmutableArray<string>.Empty);
var linkset = new VexLinkset(linksetId, tenant, vulnerabilityId, productKey, scope, Enumerable.Empty<VexLinksetObservationRefModel>());
_linksets[key] = linkset;
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);

View File

@@ -0,0 +1,13 @@
namespace StellaOps.Excititor.Core.Storage;
/// <summary>
/// Persistence abstraction for resolved VEX consensus documents.
/// </summary>
public interface IVexConsensusStore
{
ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken);
ValueTask<VexConsensus?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken);
IAsyncEnumerable<VexConsensus> FindCalculatedBeforeAsync(DateTimeOffset cutoff, int limit, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,35 @@
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Export;
/// <summary>
/// Persisted manifest store for export runs keyed by query signature and format.
/// </summary>
public interface IVexExportStore
{
ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken);
}
/// <summary>
/// Cache index used to track export cache entries by signature and format.
/// </summary>
public interface IVexCacheIndex
{
ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken);
ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
}
/// <summary>
/// Maintenance operations for keeping the export cache consistent.
/// </summary>
public interface IVexCacheMaintenance
{
ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken);
ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,206 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Npgsql;
using NpgsqlTypes;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Infrastructure.Postgres.Repositories;
namespace StellaOps.Excititor.Storage.Postgres.Repositories;
/// <summary>
/// PostgreSQL-backed connector state repository for orchestrator checkpoints and heartbeats.
/// </summary>
public sealed class PostgresConnectorStateRepository : RepositoryBase<ExcititorDataSource>, IVexConnectorStateRepository
{
private volatile bool _initialized;
private readonly SemaphoreSlim _initLock = new(1, 1);
public PostgresConnectorStateRepository(ExcititorDataSource dataSource, ILogger<PostgresConnectorStateRepository> logger)
: base(dataSource, logger)
{
}
public async ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(connectorId);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT connector_id, last_updated, document_digests, resume_tokens, last_success_at, failure_count,
next_eligible_run, last_failure_reason, last_checkpoint, last_heartbeat_at, last_heartbeat_status,
last_artifact_hash, last_artifact_kind
FROM vex.connector_states
WHERE connector_id = @connector_id;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "connector_id", connectorId);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
return null;
}
return Map(reader);
}
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(state);
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
var lastUpdated = state.LastUpdated ?? DateTimeOffset.UtcNow;
await using var connection = await DataSource.OpenConnectionAsync("public", "writer", cancellationToken).ConfigureAwait(false);
const string sql = """
INSERT INTO vex.connector_states (
connector_id, last_updated, document_digests, resume_tokens, last_success_at, failure_count,
next_eligible_run, last_failure_reason, last_checkpoint, last_heartbeat_at, last_heartbeat_status,
last_artifact_hash, last_artifact_kind)
VALUES (
@connector_id, @last_updated, @document_digests, @resume_tokens, @last_success_at, @failure_count,
@next_eligible_run, @last_failure_reason, @last_checkpoint, @last_heartbeat_at, @last_heartbeat_status,
@last_artifact_hash, @last_artifact_kind)
ON CONFLICT (connector_id) DO UPDATE SET
last_updated = EXCLUDED.last_updated,
document_digests = EXCLUDED.document_digests,
resume_tokens = EXCLUDED.resume_tokens,
last_success_at = EXCLUDED.last_success_at,
failure_count = EXCLUDED.failure_count,
next_eligible_run = EXCLUDED.next_eligible_run,
last_failure_reason = EXCLUDED.last_failure_reason,
last_checkpoint = EXCLUDED.last_checkpoint,
last_heartbeat_at = EXCLUDED.last_heartbeat_at,
last_heartbeat_status = EXCLUDED.last_heartbeat_status,
last_artifact_hash = EXCLUDED.last_artifact_hash,
last_artifact_kind = EXCLUDED.last_artifact_kind;
""";
await using var command = CreateCommand(sql, connection);
AddParameter(command, "connector_id", state.ConnectorId);
AddParameter(command, "last_updated", lastUpdated.UtcDateTime);
AddParameter(command, "document_digests", state.DocumentDigests.IsDefault ? Array.Empty<string>() : state.DocumentDigests.ToArray());
AddJsonbParameter(command, "resume_tokens", JsonSerializer.Serialize(state.ResumeTokens));
AddParameter(command, "last_success_at", state.LastSuccessAt?.UtcDateTime);
AddParameter(command, "failure_count", state.FailureCount);
AddParameter(command, "next_eligible_run", state.NextEligibleRun?.UtcDateTime);
AddParameter(command, "last_failure_reason", state.LastFailureReason);
AddParameter(command, "last_checkpoint", state.LastCheckpoint?.UtcDateTime);
AddParameter(command, "last_heartbeat_at", state.LastHeartbeatAt?.UtcDateTime);
AddParameter(command, "last_heartbeat_status", state.LastHeartbeatStatus);
AddParameter(command, "last_artifact_hash", state.LastArtifactHash);
AddParameter(command, "last_artifact_kind", state.LastArtifactKind);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
public async ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken)
{
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
await using var connection = await DataSource.OpenConnectionAsync("public", "reader", cancellationToken).ConfigureAwait(false);
const string sql = """
SELECT connector_id, last_updated, document_digests, resume_tokens, last_success_at, failure_count,
next_eligible_run, last_failure_reason, last_checkpoint, last_heartbeat_at, last_heartbeat_status,
last_artifact_hash, last_artifact_kind
FROM vex.connector_states
ORDER BY connector_id;
""";
await using var command = CreateCommand(sql, connection);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
var results = new List<VexConnectorState>();
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
results.Add(Map(reader));
}
return results;
}
private VexConnectorState Map(NpgsqlDataReader reader)
{
var connectorId = reader.GetString(0);
var lastUpdated = reader.IsDBNull(1) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(1), TimeSpan.Zero);
var digests = reader.IsDBNull(2) ? ImmutableArray<string>.Empty : reader.GetFieldValue<string[]>(2).ToImmutableArray();
var resumeTokens = reader.IsDBNull(3)
? ImmutableDictionary<string, string>.Empty
: JsonSerializer.Deserialize<ImmutableDictionary<string, string>>(reader.GetFieldValue<string>(3)) ?? ImmutableDictionary<string, string>.Empty;
var lastSuccess = reader.IsDBNull(4) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero);
var failureCount = reader.IsDBNull(5) ? 0 : reader.GetInt32(5);
var nextEligible = reader.IsDBNull(6) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(6), TimeSpan.Zero);
var lastFailureReason = reader.IsDBNull(7) ? null : reader.GetString(7);
var lastCheckpoint = reader.IsDBNull(8) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(8), TimeSpan.Zero);
var lastHeartbeatAt = reader.IsDBNull(9) ? (DateTimeOffset?)null : new DateTimeOffset(reader.GetDateTime(9), TimeSpan.Zero);
var lastHeartbeatStatus = reader.IsDBNull(10) ? null : reader.GetString(10);
var lastArtifactHash = reader.IsDBNull(11) ? null : reader.GetString(11);
var lastArtifactKind = reader.IsDBNull(12) ? null : reader.GetString(12);
return new VexConnectorState(
connectorId,
lastUpdated,
digests,
resumeTokens,
lastSuccess,
failureCount,
nextEligible,
lastFailureReason,
lastCheckpoint,
lastHeartbeatAt,
lastHeartbeatStatus,
lastArtifactHash,
lastArtifactKind);
}
private async ValueTask EnsureTableAsync(CancellationToken cancellationToken)
{
if (_initialized)
{
return;
}
await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_initialized)
{
return;
}
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
const string sql = """
CREATE TABLE IF NOT EXISTS vex.connector_states (
connector_id text PRIMARY KEY,
last_updated timestamptz NOT NULL,
document_digests text[] NOT NULL,
resume_tokens jsonb NOT NULL DEFAULT '{}'::jsonb,
last_success_at timestamptz NULL,
failure_count integer NOT NULL DEFAULT 0,
next_eligible_run timestamptz NULL,
last_failure_reason text NULL,
last_checkpoint timestamptz NULL,
last_heartbeat_at timestamptz NULL,
last_heartbeat_status text NULL,
last_artifact_hash text NULL,
last_artifact_kind text NULL
);
""";
await using var command = CreateCommand(sql, connection);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_initialized = true;
}
finally
{
_initLock.Release();
}
}
}

View File

@@ -90,8 +90,9 @@ public sealed class PostgresVexRawStore : RepositoryBase<ExcititorDataSource>, I
ON CONFLICT (digest) DO NOTHING;
""";
await using (var command = CreateCommand(insertDocumentSql, connection, transaction))
await using (var command = CreateCommand(insertDocumentSql, connection))
{
command.Transaction = transaction;
AddParameter(command, "digest", digest);
AddParameter(command, "tenant", tenant);
AddParameter(command, "provider_id", providerId);
@@ -117,7 +118,8 @@ public sealed class PostgresVexRawStore : RepositoryBase<ExcititorDataSource>, I
ON CONFLICT (digest) DO NOTHING;
""";
await using var blobCommand = CreateCommand(insertBlobSql, connection, transaction);
await using var blobCommand = CreateCommand(insertBlobSql, connection);
blobCommand.Transaction = transaction;
AddParameter(blobCommand, "digest", digest);
blobCommand.Parameters.Add(new NpgsqlParameter("payload", NpgsqlDbType.Bytea)
{
@@ -320,9 +322,15 @@ public sealed class PostgresVexRawStore : RepositoryBase<ExcititorDataSource>, I
}
private static VexDocumentFormat ParseFormat(string value)
=> Enum.TryParse<VexDocumentFormat>(value, ignoreCase: true, out var parsed)
? parsed
: VexDocumentFormat.Unknown;
{
if (Enum.TryParse<VexDocumentFormat>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
// Default to OpenVEX for unknown/legacy values to preserve compatibility with legacy rows.
return VexDocumentFormat.OpenVex;
}
private static ImmutableDictionary<string, string> ParseMetadata(string json)
{

View File

@@ -34,6 +34,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<IAppendOnlyLinksetStore, PostgresAppendOnlyLinksetStore>();
services.AddScoped<IVexLinksetStore, PostgresAppendOnlyLinksetStore>();
services.AddScoped<IVexRawStore, PostgresVexRawStore>();
services.AddScoped<IVexConnectorStateRepository, PostgresConnectorStateRepository>();
return services;
}
@@ -56,6 +57,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<IAppendOnlyLinksetStore, PostgresAppendOnlyLinksetStore>();
services.AddScoped<IVexLinksetStore, PostgresAppendOnlyLinksetStore>();
services.AddScoped<IVexRawStore, PostgresVexRawStore>();
services.AddScoped<IVexConnectorStateRepository, PostgresConnectorStateRepository>();
return services;
}