stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 16 Releases Wiki Activity

feat: Add documentation and task tracking for Sprints 508 to 514 in Ops & Offline

Browse Source 
- Created detailed markdown files for Sprints 508 (Ops Offline Kit), 509 (Samples), 510 (AirGap), 511 (Api), 512 (Bench), 513 (Provenance), and 514 (Sovereign Crypto Enablement) outlining tasks, dependencies, and owners.
- Introduced a comprehensive Reachability Evidence Delivery Guide to streamline the reachability signal process.
- Implemented unit tests for Advisory AI to block known injection patterns and redact secrets.
- Added AuthoritySenderConstraintHelper to manage sender constraints in OpenIddict transactions.
This commit is contained in:
master
2025-11-08 23:18:28 +02:00
parent 536f6249a6
commit ae69b1a8a1
187 changed files with 4326 additions and 3196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs
View File

@@ -22,45 +22,45 @@ public sealed class ConcelierOptions
public AdvisoryChunkOptions AdvisoryChunks { get; set; } = new();
public StellaOpsCryptoOptions Crypto { get; } = new();
public sealed class StorageOptions
{
public string Driver { get; set; } = "mongo";
public string Dsn { get; set; } = string.Empty;
public string? Database { get; set; }
public int CommandTimeoutSeconds { get; set; } = 30;
}
public sealed class PluginOptions
{
public string? BaseDirectory { get; set; }
public string? Directory { get; set; }
public IList<string> SearchPatterns { get; set; } = new List<string>();
}
public sealed class TelemetryOptions
{
public bool Enabled { get; set; } = true;
public bool EnableTracing { get; set; } = true;
public bool EnableMetrics { get; set; } = true;
public bool EnableLogging { get; set; } = true;
public string MinimumLogLevel { get; set; } = "Information";
public string? ServiceName { get; set; }
public string? OtlpEndpoint { get; set; }
public IDictionary<string, string> OtlpHeaders { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public sealed class StorageOptions
{
public string Driver { get; set; } = "mongo";
public string Dsn { get; set; } = string.Empty;
public string? Database { get; set; }
public int CommandTimeoutSeconds { get; set; } = 30;
}
public sealed class PluginOptions
{
public string? BaseDirectory { get; set; }
public string? Directory { get; set; }
public IList<string> SearchPatterns { get; set; } = new List<string>();
}
public sealed class TelemetryOptions
{
public bool Enabled { get; set; } = true;
public bool EnableTracing { get; set; } = true;
public bool EnableMetrics { get; set; } = true;
public bool EnableLogging { get; set; } = true;
public string MinimumLogLevel { get; set; } = "Information";
public string? ServiceName { get; set; }
public string? OtlpEndpoint { get; set; }
public IDictionary<string, string> OtlpHeaders { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public IDictionary<string, string> ResourceAttributes { get; set; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public bool ExportConsole { get; set; }
@@ -96,6 +96,8 @@ public sealed class ConcelierOptions
public string? ClientSecretFile { get; set; }
public string? TestSigningSecret { get; set; }
public IList<string> ClientScopes { get; set; } = new List<string>();
public ResilienceOptions Resilience { get; set; } = new();

210
src/Concelier/StellaOps.Concelier.WebService/Program.cs
View File

@@ -5,6 +5,8 @@ using System.Globalization;
using System.Linq;
using System.Security.Claims;
using System.Text;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.IdentityModel.Tokens;
using Microsoft.AspNetCore.Diagnostics;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
@@ -16,6 +18,7 @@ using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Primitives;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Events;
@@ -101,6 +104,7 @@ builder.Services.AddConcelierAocGuards();
builder.Services.AddConcelierLinksetMappers();
builder.Services.AddAdvisoryRawServices();
builder.Services.AddSingleton<IAdvisoryObservationQueryService, AdvisoryObservationQueryService>();
builder.Services.AddSingleton<AdvisoryChunkBuilder>();
var features = concelierOptions.Features ?? new ConcelierOptions.FeaturesOptions();
@@ -139,6 +143,7 @@ builder.Services.AddAocGuard();
var authorityConfigured = concelierOptions.Authority is { Enabled: true };
if (authorityConfigured)
{
builder.Services.AddStellaOpsAuthClient(clientOptions =>
@@ -180,36 +185,61 @@ if (authorityConfigured)
}
});
builder.Services.AddStellaOpsResourceServerAuthentication(
builder.Configuration,
configurationSection: null,
configure: resourceOptions =>
{
resourceOptions.Authority = concelierOptions.Authority.Issuer;
resourceOptions.RequireHttpsMetadata = concelierOptions.Authority.RequireHttpsMetadata;
resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(concelierOptions.Authority.BackchannelTimeoutSeconds);
resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(concelierOptions.Authority.TokenClockSkewSeconds);
if (!string.IsNullOrWhiteSpace(concelierOptions.Authority.MetadataAddress))
if (string.IsNullOrWhiteSpace(concelierOptions.Authority.TestSigningSecret))
{
builder.Services.AddStellaOpsResourceServerAuthentication(
builder.Configuration,
configurationSection: null,
configure: resourceOptions =>
{
resourceOptions.MetadataAddress = concelierOptions.Authority.MetadataAddress;
}
resourceOptions.Authority = concelierOptions.Authority.Issuer;
resourceOptions.RequireHttpsMetadata = concelierOptions.Authority.RequireHttpsMetadata;
resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(concelierOptions.Authority.BackchannelTimeoutSeconds);
resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(concelierOptions.Authority.TokenClockSkewSeconds);
foreach (var audience in concelierOptions.Authority.Audiences)
{
resourceOptions.Audiences.Add(audience);
}
if (!string.IsNullOrWhiteSpace(concelierOptions.Authority.MetadataAddress))
{
resourceOptions.MetadataAddress = concelierOptions.Authority.MetadataAddress;
}
foreach (var scope in concelierOptions.Authority.RequiredScopes)
{
resourceOptions.RequiredScopes.Add(scope);
}
foreach (var audience in concelierOptions.Authority.Audiences)
{
resourceOptions.Audiences.Add(audience);
}
foreach (var network in concelierOptions.Authority.BypassNetworks)
foreach (var scope in concelierOptions.Authority.RequiredScopes)
{
resourceOptions.RequiredScopes.Add(scope);
}
foreach (var network in concelierOptions.Authority.BypassNetworks)
{
resourceOptions.BypassNetworks.Add(network);
}
});
}
else
{
builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
.AddJwtBearer(options =>
{
resourceOptions.BypassNetworks.Add(network);
}
});
options.RequireHttpsMetadata = concelierOptions.Authority.RequireHttpsMetadata;
options.TokenValidationParameters = new TokenValidationParameters
{
ValidateIssuerSigningKey = true,
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(concelierOptions.Authority.TestSigningSecret!)),
ValidateIssuer = true,
ValidIssuer = concelierOptions.Authority.Issuer,
ValidateAudience = concelierOptions.Authority.Audiences.Count > 0,
ValidAudiences = concelierOptions.Authority.Audiences,
ValidateLifetime = true,
ClockSkew = TimeSpan.FromSeconds(concelierOptions.Authority.TokenClockSkewSeconds),
NameClaimType = StellaOpsClaimTypes.Subject,
RoleClaimType = ClaimTypes.Role
};
});
}
}
builder.Services.AddAuthorization(options =>
{
@@ -250,6 +280,12 @@ if (resolvedAuthority.Enabled && resolvedAuthority.AllowAnonymousFallback)
"Authority authentication is configured but anonymous fallback remains enabled. Set authority.allowAnonymousFallback to false before 2025-12-31 to complete the rollout.");
}
if (authorityConfigured)
{
app.UseAuthentication();
app.UseAuthorization();
}
app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority);
app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
@@ -689,16 +725,18 @@ var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKe
return Problem(context, "advisoryKey is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide an advisory identifier.");
}
var normalizedKey = advisoryKey.Trim();
var canonicalKey = normalizedKey.ToUpperInvariant();
var vendorFilter = AdvisoryRawRequestMapper.NormalizeStrings(context.Request.Query["vendor"]);
var records = await rawService.FindByAdvisoryKeyAsync(
tenant,
advisoryKey,
canonicalKey,
vendorFilter,
cancellationToken).ConfigureAwait(false);
if (records.Count == 0)
{
return Results.NotFound();
return Problem(context, "Advisory not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"No evidence available for {normalizedKey}.");
}
var recordResponses = records
@@ -710,7 +748,8 @@ var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKe
record.Document))
.ToArray();
var response = new AdvisoryEvidenceResponse(recordResponses[0].Document.AdvisoryKey, recordResponses);
var responseKey = recordResponses[0].Document.AdvisoryKey ?? canonicalKey;
var response = new AdvisoryEvidenceResponse(responseKey, recordResponses);
return JsonResult(response);
});
if (authorityConfigured)
@@ -718,6 +757,67 @@ if (authorityConfigured)
advisoryEvidenceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
var advisoryChunksEndpoint = app.MapGet("/advisories/{advisoryKey}/chunks", async (
string advisoryKey,
HttpContext context,
[FromServices] IAdvisoryObservationQueryService observationService,
[FromServices] AdvisoryChunkBuilder chunkBuilder,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var authorizationError = EnsureTenantAuthorized(context, tenant);
if (authorizationError is not null)
{
return authorizationError;
}
if (string.IsNullOrWhiteSpace(advisoryKey))
{
return Problem(context, "advisoryKey is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide an advisory identifier.");
}
var normalizedKey = advisoryKey.Trim();
var chunkSettings = resolvedConcelierOptions.AdvisoryChunks ?? new ConcelierOptions.AdvisoryChunkOptions();
var chunkLimit = ResolveBoundedInt(context.Request.Query["limit"], chunkSettings.DefaultChunkLimit, 1, chunkSettings.MaxChunkLimit);
var observationLimit = ResolveBoundedInt(context.Request.Query["observations"], chunkSettings.DefaultObservationLimit, 1, chunkSettings.MaxObservationLimit);
var minimumLength = ResolveBoundedInt(context.Request.Query["minLength"], chunkSettings.DefaultMinimumLength, 16, chunkSettings.MaxMinimumLength);
var sectionFilter = BuildFilterSet(context.Request.Query["section"]);
var formatFilter = BuildFilterSet(context.Request.Query["format"]);
var queryOptions = new AdvisoryObservationQueryOptions(
tenant,
aliases: new[] { normalizedKey },
limit: observationLimit);
var observationResult = await observationService.QueryAsync(queryOptions, cancellationToken).ConfigureAwait(false);
if (observationResult.Observations.IsDefaultOrEmpty || observationResult.Observations.Length == 0)
{
return Problem(context, "Advisory not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"No observations available for {normalizedKey}.");
}
var buildOptions = new AdvisoryChunkBuildOptions(
normalizedKey,
chunkLimit,
observationLimit,
sectionFilter,
formatFilter,
minimumLength);
var response = chunkBuilder.Build(buildOptions, observationResult.Observations.ToArray());
return JsonResult(response);
});
if (authorityConfigured)
{
advisoryChunksEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
var aocVerifyEndpoint = app.MapPost("/aoc/verify", async (
HttpContext context,
AocVerifyRequest request,
@@ -932,12 +1032,6 @@ if (authorityConfigured)
});
}
if (authorityConfigured)
{
app.UseAuthentication();
app.UseAuthorization();
}
IResult JsonResult<T>(T value, int? statusCode = null)
{
var payload = JsonSerializer.Serialize(value, jsonOptions);
@@ -1049,6 +1143,53 @@ IResult? EnsureTenantAuthorized(HttpContext context, string tenant)
return null;
}
ImmutableHashSet<string> BuildFilterSet(StringValues values)
{
if (values.Count == 0)
{
return ImmutableHashSet<string>.Empty;
}
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
var segments = value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (segments.Length == 0)
{
builder.Add(value.Trim());
continue;
}
foreach (var segment in segments)
{
if (!string.IsNullOrWhiteSpace(segment))
{
builder.Add(segment.Trim());
}
}
}
return builder.ToImmutable();
}
int ResolveBoundedInt(StringValues values, int fallback, int minValue, int maxValue)
{
foreach (var value in values)
{
if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
{
return Math.Clamp(parsed, minValue, maxValue);
}
}
return Math.Clamp(fallback, minValue, maxValue);
}
static DateTimeOffset? ParseDateTime(string? value)
{
if (string.IsNullOrWhiteSpace(value))
@@ -1474,3 +1615,4 @@ static async Task InitializeMongoAsync(WebApplication app)
}
public partial class Program;

16
src/Concelier/StellaOps.Concelier.WebService/Services/AdvisoryChunkBuilder.cs
View File

@@ -1,11 +1,11 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.WebService.Contracts;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.WebService.Services;
@@ -20,6 +20,12 @@ internal sealed record AdvisoryChunkBuildOptions(
internal sealed class AdvisoryChunkBuilder
{
private const int DefaultMinLength = 40;
private readonly ICryptoHash _hash;
public AdvisoryChunkBuilder(ICryptoHash hash)
{
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
}
public AdvisoryChunkCollectionResponse Build(
AdvisoryChunkBuildOptions options,
@@ -97,7 +103,7 @@ internal sealed class AdvisoryChunkBuilder
return observation.ObservationId;
}
private static IEnumerable<AdvisoryChunkItemResponse> ExtractChunks(
private IEnumerable<AdvisoryChunkItemResponse> ExtractChunks(
AdvisoryObservation observation,
string documentId,
AdvisoryChunkBuildOptions options)
@@ -248,10 +254,10 @@ internal sealed class AdvisoryChunkBuilder
return string.Concat(path, '[', index.ToString(CultureInfo.InvariantCulture), ']');
}
private static string CreateChunkId(string documentId, string paragraphId)
private string CreateChunkId(string documentId, string paragraphId)
{
var input = string.Concat(documentId, '|', paragraphId);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input));
return string.Concat(documentId, ':', Convert.ToHexString(hash.AsSpan(0, 8)));
var digest = _hash.ComputeHash(Encoding.UTF8.GetBytes(input), HashAlgorithms.Sha256);
return string.Concat(documentId, ':', Convert.ToHexString(digest.AsSpan(0, 8)));
}
}

8
src/Concelier/StellaOps.Concelier.WebService/TASKS.md
View File

@@ -10,10 +10,10 @@
> Docs alignment (2025-10-26): Guard rules + error codes documented in AOC reference §5 and CLI guide.
| CONCELIER-WEB-AOC-19-004 `End-to-end ingest verification` | TODO | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-003, CONCELIER-CORE-AOC-19-002 | Create integration tests ingesting large advisory batches (cold/warm) validating linkset enrichment, metrics emission, and reproducible outputs. Capture load-test scripts + doc notes for Offline Kit dry runs. |
> Docs alignment (2025-10-26): Offline verification workflow referenced in `docs/deploy/containers.md` §5.
| CONCELIER-WEB-AOC-19-005 `Chunk evidence regression` | TODO (2025-11-08) | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-002 | Fix `/advisories/{key}/chunks` seeded fixtures so AdvisoryChunksEndpoint tests stop returning 404/not-found when raw documents are pre-populated; ensure Mongo migrations no longer emit “Unable to locate advisory_raw documents” during test boot. |
| CONCELIER-WEB-AOC-19-006 `Allowlist ingest auth parity` | TODO (2025-11-08) | Concelier WebService Guild | CONCELIER-WEB-AOC-19-002 | Align WebService auth defaults with the test tokens so the allowlisted tenant can create an advisory before forbidden tenants are rejected in `AdvisoryIngestEndpoint_RejectsTenantOutsideAllowlist`. |
| CONCELIER-WEB-AOC-19-007 `AOC verify violation codes` | TODO (2025-11-08) | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-002 | Update AOC verify logic/fixtures so guard failures produce the expected `ERR_AOC_001` payload (current regression returns `ERR_AOC_004`) while keeping the mapper/guard parity exercised by the new tests. |
| CONCELIER-CRYPTO-90-001 `Crypto provider adoption` | DOING (2025-11-08) | Concelier WebService Guild, Security Guild | SEC-CRYPTO-90-003, SEC-CRYPTO-90-004 | Route hashing/signing in OpenAPI discovery, Mirror connectors, and RU advisory adapters through `ICryptoProviderRegistry` so RootPack_RU uses CryptoPro/PKCS#11 keys. Reference `docs/security/crypto-routing-audit-2025-11-07.md`. |
| CONCELIER-WEB-AOC-19-005 `Chunk evidence regression` | DOING (2025-11-08) | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-002 | Fix `/advisories/{key}/chunks` seeded fixtures so AdvisoryChunksEndpoint tests stop returning 404/not-found when raw documents are pre-populated; ensure Mongo migrations no longer emit “Unable to locate advisory_raw documents” during test boot. |
| CONCELIER-WEB-AOC-19-006 `Allowlist ingest auth parity` | DOING (2025-11-08) | Concelier WebService Guild | CONCELIER-WEB-AOC-19-002 | Align WebService auth defaults with the test tokens so the allowlisted tenant can create an advisory before forbidden tenants are rejected in `AdvisoryIngestEndpoint_RejectsTenantOutsideAllowlist`. |
| CONCELIER-WEB-AOC-19-007 `AOC verify violation codes` | DOING (2025-11-08) | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-002 | Update AOC verify logic/fixtures so guard failures produce the expected `ERR_AOC_001` payload (current regression returns `ERR_AOC_004`) while keeping the mapper/guard parity exercised by the new tests. |
| CONCELIER-CRYPTO-90-001 `Crypto provider adoption` | DONE (2025-11-08) | Concelier WebService Guild, Security Guild | SEC-CRYPTO-90-003, SEC-CRYPTO-90-004 | OpenAPI discovery, chunk builders, SourceFetchService, SourceStateSeedProcessor, and all distro/OSV/NVD connectors now route hashing through `ICryptoHash` so RootPack_RU can swap CryptoPro/PKCS#11 providers. Reference `docs/security/crypto-routing-audit-2025-11-07.md`. |
## Policy Engine v2

19
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs
View File

@@ -6,7 +6,6 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -19,6 +18,7 @@ using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using System.Text.Json;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
@@ -40,15 +40,17 @@ public sealed class SourceFetchService
private readonly IAdvisoryRawWriteGuard _guard;
private readonly IAdvisoryLinksetMapper _linksetMapper;
private readonly string _connectorVersion;
public SourceFetchService(
IHttpClientFactory httpClientFactory,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
ILogger<SourceFetchService> logger,
private readonly ICryptoHash _hash;
public SourceFetchService(
IHttpClientFactory httpClientFactory,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
ILogger<SourceFetchService> logger,
IJitterSource jitterSource,
IAdvisoryRawWriteGuard guard,
IAdvisoryLinksetMapper linksetMapper,
ICryptoHash hash,
TimeProvider? timeProvider = null,
IOptionsMonitor<SourceHttpClientOptions>? httpClientOptions = null,
IOptions<MongoStorageOptions>? storageOptions = null)
@@ -60,6 +62,7 @@ public sealed class SourceFetchService
_jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource));
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
_linksetMapper = linksetMapper ?? throw new ArgumentNullException(nameof(linksetMapper));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_timeProvider = timeProvider ?? TimeProvider.System;
_httpClientOptions = httpClientOptions ?? throw new ArgumentNullException(nameof(httpClientOptions));
_storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions));
@@ -103,7 +106,7 @@ public sealed class SourceFetchService
}
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var contentHash = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant();
var contentHash = _hash.ComputeHashHex(contentBytes, HashAlgorithms.Sha256);
var fetchedAt = _timeProvider.GetUtcNow();
var contentType = response.Content.Headers.ContentType?.ToString();

71
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs
View File

@@ -1,10 +1,10 @@
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Common.State;
@@ -15,23 +15,26 @@ public sealed class SourceStateSeedProcessor
{
private readonly IDocumentStore _documentStore;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly ISourceStateRepository _stateRepository;
private readonly TimeProvider _timeProvider;
private readonly ILogger<SourceStateSeedProcessor> _logger;
public SourceStateSeedProcessor(
IDocumentStore documentStore,
RawDocumentStorage rawDocumentStorage,
ISourceStateRepository stateRepository,
TimeProvider? timeProvider = null,
ILogger<SourceStateSeedProcessor>? logger = null)
{
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? NullLogger<SourceStateSeedProcessor>.Instance;
}
private readonly ISourceStateRepository _stateRepository;
private readonly TimeProvider _timeProvider;
private readonly ILogger<SourceStateSeedProcessor> _logger;
private readonly ICryptoHash _hash;
public SourceStateSeedProcessor(
IDocumentStore documentStore,
RawDocumentStorage rawDocumentStorage,
ISourceStateRepository stateRepository,
ICryptoHash hash,
TimeProvider? timeProvider = null,
ILogger<SourceStateSeedProcessor>? logger = null)
{
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? NullLogger<SourceStateSeedProcessor>.Instance;
}
public async Task<SourceStateSeedResult> ProcessAsync(SourceStateSeedSpecification specification, CancellationToken cancellationToken)
{
@@ -138,7 +141,7 @@ public sealed class SourceStateSeedProcessor
_logger.LogWarning("Seed document URI '{Uri}' does not appear to be absolute.", document.Uri);
}
var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
var contentHash = _hash.ComputeHashHex(payload, HashAlgorithms.Sha256);
var existing = await _documentStore.FindBySourceAndUriAsync(source, document.Uri, cancellationToken).ConfigureAwait(false);
@@ -168,12 +171,12 @@ public sealed class SourceStateSeedProcessor
var metadata = CloneDictionary(document.Metadata);
var record = new DocumentRecord(
document.DocumentId ?? existing?.Id ?? Guid.NewGuid(),
source,
document.Uri,
document.FetchedAt ?? completedAt,
sha256,
var record = new DocumentRecord(
document.DocumentId ?? existing?.Id ?? Guid.NewGuid(),
source,
document.Uri,
document.FetchedAt ?? completedAt,
contentHash,
string.IsNullOrWhiteSpace(document.Status) ? DocumentStatuses.PendingParse : document.Status,
document.ContentType,
headers,
@@ -181,9 +184,9 @@ public sealed class SourceStateSeedProcessor
document.Etag,
document.LastModified,
gridId,
document.ExpiresAt);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
document.ExpiresAt);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
documentIds.Add(upserted.Id);

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj
View File

@@ -17,6 +17,7 @@
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
</ItemGroup>
</Project>
</Project>

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Ubuntu/StellaOps.Concelier.Connector.Distro.Ubuntu.csproj
View File

@@ -14,5 +14,6 @@
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
</ItemGroup>
</Project>
</Project>

57
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnector.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.Text;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using System.Globalization;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
@@ -29,8 +29,9 @@ public sealed class UbuntuConnector : IFeedConnector
private readonly IAdvisoryStore _advisoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly UbuntuOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<UbuntuConnector> _logger;
private readonly TimeProvider _timeProvider;
private readonly ILogger<UbuntuConnector> _logger;
private readonly ICryptoHash _hash;
private static readonly Action<ILogger, string, int, Exception?> LogMapped =
LoggerMessage.Define<string, int>(
@@ -45,9 +46,10 @@ public sealed class UbuntuConnector : IFeedConnector
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
IOptions<UbuntuOptions> options,
TimeProvider? timeProvider,
ILogger<UbuntuConnector> logger)
IOptions<UbuntuOptions> options,
TimeProvider? timeProvider,
ILogger<UbuntuConnector> logger,
ICryptoHash cryptoHash)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
@@ -56,9 +58,10 @@ public sealed class UbuntuConnector : IFeedConnector
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_hash = cryptoHash ?? throw new ArgumentNullException(nameof(cryptoHash));
}
public string SourceName => UbuntuConnectorPlugin.SourceName;
@@ -418,9 +421,9 @@ public sealed class UbuntuConnector : IFeedConnector
private static string ComputeNoticeHash(BsonDocument document)
{
var bytes = document.ToBson();
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
var hash = _hash.ComputeHash(bytes, HashAlgorithms.Sha256);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static BsonDocument ToBson(UbuntuNoticeDto notice)
{

45
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs
View File

@@ -1,5 +1,4 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
@@ -17,8 +16,9 @@ using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Storage.Mongo.ChangeHistory;
using StellaOps.Plugin;
using Json.Schema;
using StellaOps.Plugin;
using Json.Schema;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Nvd;
@@ -32,10 +32,11 @@ public sealed class NvdConnector : IFeedConnector
private readonly IChangeHistoryStore _changeHistoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly IJsonSchemaValidator _schemaValidator;
private readonly NvdOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NvdConnector> _logger;
private readonly NvdDiagnostics _diagnostics;
private readonly NvdOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NvdConnector> _logger;
private readonly NvdDiagnostics _diagnostics;
private readonly ICryptoHash _hash;
private static readonly JsonSchema Schema = NvdSchemaProvider.Schema;
@@ -48,10 +49,11 @@ public sealed class NvdConnector : IFeedConnector
IChangeHistoryStore changeHistoryStore,
ISourceStateRepository stateRepository,
IJsonSchemaValidator schemaValidator,
IOptions<NvdOptions> options,
NvdDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<NvdConnector> logger)
IOptions<NvdOptions> options,
NvdDiagnostics diagnostics,
ICryptoHash hash,
TimeProvider? timeProvider,
ILogger<NvdConnector> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
@@ -63,10 +65,11 @@ public sealed class NvdConnector : IFeedConnector
_schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => NvdConnectorPlugin.SourceName;
@@ -524,12 +527,12 @@ public sealed class NvdConnector : IFeedConnector
private static string SerializeElement(JsonElement element)
=> JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false });
private static string ComputeHash(string snapshot)
{
var bytes = Encoding.UTF8.GetBytes(snapshot);
var hash = SHA256.HashData(bytes);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private string ComputeHash(string snapshot)
{
var bytes = Encoding.UTF8.GetBytes(snapshot);
var hex = _hash.ComputeHashHex(bytes, HashAlgorithms.Sha256);
return $"sha256:{hex}";
}
private async Task<NvdCursor> GetCursorAsync(CancellationToken cancellationToken)
{

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/StellaOps.Concelier.Connector.Nvd.csproj
View File

@@ -9,10 +9,11 @@
<EmbeddedResource Include="Schemas\nvd-vulnerability.schema.json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
</ItemGroup>
</Project>
</Project>

17
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs
View File

@@ -5,9 +5,8 @@ using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -21,9 +20,10 @@ using StellaOps.Concelier.Connector.Osv.Configuration;
using StellaOps.Concelier.Connector.Osv.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Osv;
@@ -45,6 +45,7 @@ public sealed class OsvConnector : IFeedConnector
private readonly TimeProvider _timeProvider;
private readonly ILogger<OsvConnector> _logger;
private readonly OsvDiagnostics _diagnostics;
private readonly ICryptoHash _hash;
public OsvConnector(
IHttpClientFactory httpClientFactory,
@@ -55,6 +56,7 @@ public sealed class OsvConnector : IFeedConnector
ISourceStateRepository stateRepository,
IOptions<OsvOptions> options,
OsvDiagnostics diagnostics,
ICryptoHash hash,
TimeProvider? timeProvider,
ILogger<OsvConnector> logger)
{
@@ -66,6 +68,7 @@ public sealed class OsvConnector : IFeedConnector
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -415,7 +418,7 @@ public sealed class OsvConnector : IFeedConnector
}
var documentUri = BuildDocumentUri(ecosystem, dto.Id);
var sha256 = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
var sha256 = _hash.ComputeHashHex(bytes, HashAlgorithms.Sha256);
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
if (existing is not null && string.Equals(existing.Sha256, sha256, StringComparison.OrdinalIgnoreCase))

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj
View File

@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
@@ -21,4 +22,4 @@
<_Parameter1>StellaOps.Concelier.Connector.Osv.Tests</_Parameter1>
</AssemblyAttribute>
</ItemGroup>
</Project>
</Project>

64
src/Concelier/__Tests/StellaOps.Concelier.Connector.Common.Tests/Common/SourceFetchServiceGuardTests.cs
View File

@@ -1,8 +1,7 @@
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Text;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Mongo2Go;
@@ -13,9 +12,10 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Core.Aoc;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Tests;
@@ -23,14 +23,16 @@ public sealed class SourceFetchServiceGuardTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
private readonly RawDocumentStorage _rawStorage;
private readonly RawDocumentStorage _rawStorage;
private readonly ICryptoHash _hash;
public SourceFetchServiceGuardTests()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase($"source-fetch-guard-{Guid.NewGuid():N}");
_rawStorage = new RawDocumentStorage(_database);
_rawStorage = new RawDocumentStorage(_database);
_hash = CryptoHashFactory.CreateDefault();
}
[Fact]
@@ -53,17 +55,18 @@ public sealed class SourceFetchServiceGuardTests : IAsyncLifetime
var linksetMapper = new NoopAdvisoryLinksetMapper();
var service = new SourceFetchService(
httpClientFactory,
_rawStorage,
documentStore,
NullLogger<SourceFetchService>.Instance,
jitter,
guard,
linksetMapper,
TimeProvider.System,
httpOptions,
storageOptions);
var service = new SourceFetchService(
httpClientFactory,
_rawStorage,
documentStore,
NullLogger<SourceFetchService>.Instance,
jitter,
guard,
linksetMapper,
_hash,
TimeProvider.System,
httpOptions,
storageOptions);
var request = new SourceFetchRequest("client", "vndr.msrc", new Uri("https://example.test/advisories/ADV-1234"))
{
@@ -82,7 +85,7 @@ public sealed class SourceFetchServiceGuardTests : IAsyncLifetime
Assert.Equal("tenant-default", guard.LastDocument!.Tenant);
Assert.Equal("msrc", guard.LastDocument.Source.Vendor);
Assert.Equal("ADV-1234", guard.LastDocument.Upstream.UpstreamId);
var expectedHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(responsePayload))).ToLowerInvariant();
var expectedHash = _hash.ComputeHashHex(Encoding.UTF8.GetBytes(responsePayload), HashAlgorithms.Sha256);
Assert.Equal(expectedHash, guard.LastDocument.Upstream.ContentHash);
Assert.NotNull(documentStore.LastRecord);
Assert.True(documentStore.UpsertCount > 0);
@@ -114,17 +117,18 @@ public sealed class SourceFetchServiceGuardTests : IAsyncLifetime
var linksetMapper = new NoopAdvisoryLinksetMapper();
var service = new SourceFetchService(
httpClientFactory,
_rawStorage,
documentStore,
NullLogger<SourceFetchService>.Instance,
jitter,
guard,
linksetMapper,
TimeProvider.System,
httpOptions,
storageOptions);
var service = new SourceFetchService(
httpClientFactory,
_rawStorage,
documentStore,
NullLogger<SourceFetchService>.Instance,
jitter,
guard,
linksetMapper,
_hash,
TimeProvider.System,
httpOptions,
storageOptions);
var request = new SourceFetchRequest("client", "nvd", new Uri("https://example.test/data/XYZ"))
{

4
src/Concelier/__Tests/StellaOps.Concelier.Connector.Common.Tests/Common/SourceStateSeedProcessorTests.cs
View File

@@ -11,6 +11,7 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.State;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Tests;
@@ -23,6 +24,7 @@ public sealed class SourceStateSeedProcessorTests : IAsyncLifetime
private readonly RawDocumentStorage _rawStorage;
private readonly MongoSourceStateRepository _stateRepository;
private readonly FakeTimeProvider _timeProvider;
private readonly ICryptoHash _hash;
public SourceStateSeedProcessorTests()
{
@@ -33,6 +35,7 @@ public sealed class SourceStateSeedProcessorTests : IAsyncLifetime
_rawStorage = new RawDocumentStorage(_database);
_stateRepository = new MongoSourceStateRepository(_database, NullLogger<MongoSourceStateRepository>.Instance);
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 28, 12, 0, 0, TimeSpan.Zero));
_hash = CryptoHashFactory.CreateDefault();
}
[Fact]
@@ -199,6 +202,7 @@ public sealed class SourceStateSeedProcessorTests : IAsyncLifetime
_documentStore,
_rawStorage,
_stateRepository,
_hash,
_timeProvider,
NullLogger<SourceStateSeedProcessor>.Instance);

1
src/Concelier/__Tests/StellaOps.Concelier.Connector.Common.Tests/StellaOps.Concelier.Connector.Common.Tests.csproj
View File

@@ -20,5 +20,6 @@
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
</ItemGroup>
</Project>

3
src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests.csproj
View File

@@ -10,10 +10,11 @@
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography.DependencyInjection/StellaOps.Cryptography.DependencyInjection.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="Fixtures\**\*">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
</Project>

24
src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs
View File

@@ -20,7 +20,8 @@ using StellaOps.Concelier.Connector.Distro.Ubuntu;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Testing;
using StellaOps.Concelier.Testing;
using StellaOps.Cryptography.DependencyInjection;
using Xunit;
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Tests;
@@ -94,16 +95,17 @@ public sealed class UbuntuConnectorTests : IAsyncLifetime
services.AddSingleton<TimeProvider>(_timeProvider);
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddUbuntuConnector(options =>
{
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddStellaOpsCrypto();
services.AddUbuntuConnector(options =>
{
options.NoticesEndpoint = new Uri("https://ubuntu.com/security/notices.json");
options.NoticeDetailBaseUri = new Uri("https://ubuntu.com/security/");
options.MaxNoticesPerFetch = 2;

29
src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs
View File

@@ -5,24 +5,25 @@ using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.RegularExpressions;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Osv;
using StellaOps.Concelier.Connector.Osv.Internal;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using Xunit;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Osv;
using StellaOps.Concelier.Connector.Osv.Internal;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Cryptography;
using Xunit;
namespace StellaOps.Concelier.Connector.Osv.Tests;
public sealed class OsvGhsaParityRegressionTests
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
public sealed class OsvGhsaParityRegressionTests
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
private static readonly ICryptoHash Hash = CryptoHashFactory.CreateDefault();
// Curated GHSA identifiers spanning multiple ecosystems (PyPI, npm/go, Maven) for parity coverage.
private static readonly string[] GhsaIds =
@@ -560,7 +561,7 @@ public sealed class OsvGhsaParityRegressionTests
private static string ComputeSha256Hex(string payload)
{
var bytes = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(payload));
var bytes = Hash.ComputeHash(System.Text.Encoding.UTF8.GetBytes(payload), HashAlgorithms.Sha256);
return Convert.ToHexString(bytes);
}

3
src/Concelier/__Tests/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj
View File

@@ -10,10 +10,11 @@
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="Fixtures\*.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
</Project>

25
src/Concelier/__Tests/StellaOps.Concelier.Storage.Mongo.Tests/MongoAdvisoryEventRepositoryTests.cs
View File

@@ -1,8 +1,7 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
@@ -11,17 +10,19 @@ using StellaOps.Concelier.Core.Events;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Conflicts;
using StellaOps.Concelier.Storage.Mongo.Events;
using StellaOps.Concelier.Storage.Mongo.Statements;
using StellaOps.Concelier.Testing;
using Xunit;
using StellaOps.Concelier.Storage.Mongo.Statements;
using StellaOps.Concelier.Testing;
using StellaOps.Cryptography;
using Xunit;
namespace StellaOps.Concelier.Storage.Mongo.Tests;
[Collection("mongo-fixture")]
public sealed class MongoAdvisoryEventRepositoryTests
{
private readonly IMongoDatabase _database;
private readonly MongoAdvisoryEventRepository _repository;
public sealed class MongoAdvisoryEventRepositoryTests
{
private readonly IMongoDatabase _database;
private readonly MongoAdvisoryEventRepository _repository;
private static readonly ICryptoHash Hash = CryptoHashFactory.CreateDefault();
public MongoAdvisoryEventRepositoryTests(MongoIntegrationFixture fixture)
{
@@ -36,7 +37,8 @@ public sealed class MongoAdvisoryEventRepositoryTests
{
var advisory = CreateSampleAdvisory("CVE-2025-7777", "Sample advisory");
var canonicalJson = CanonicalJsonSerializer.Serialize(advisory);
var hash = ImmutableArray.Create(SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson)));
var digest = Hash.ComputeHash(Encoding.UTF8.GetBytes(canonicalJson), HashAlgorithms.Sha256);
var hash = ImmutableArray.Create(digest);
var entry = new AdvisoryStatementEntry(
Guid.NewGuid(),
@@ -62,7 +64,8 @@ public sealed class MongoAdvisoryEventRepositoryTests
public async Task InsertAndFetchConflicts_PreservesDetails()
{
var detailJson = CanonicalJsonSerializer.Serialize(new ConflictPayload("severity", "mismatch"));
var hash = ImmutableArray.Create(SHA256.HashData(Encoding.UTF8.GetBytes(detailJson)));
var digest = Hash.ComputeHash(Encoding.UTF8.GetBytes(detailJson), HashAlgorithms.Sha256);
var hash = ImmutableArray.Create(digest);
var statementIds = ImmutableArray.Create(Guid.NewGuid(), Guid.NewGuid());
var entry = new AdvisoryConflictEntry(

3
src/Concelier/__Tests/StellaOps.Concelier.Storage.Mongo.Tests/StellaOps.Concelier.Storage.Mongo.Tests.csproj
View File

@@ -12,5 +12,6 @@
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
</ItemGroup>
</Project>
</Project>

1
src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj
View File

@@ -10,6 +10,7 @@
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="../../__Analyzers/StellaOps.Concelier.Merge.Analyzers/StellaOps.Concelier.Merge.Analyzers.csproj"
OutputItemType="Analyzer"
ReferenceOutputAssembly="false" />

18
src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs
View File

@@ -9,7 +9,6 @@ using System.Net;
using System.Net.Http.Json;
using System.Net.Http.Headers;
using System.Security.Claims;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
@@ -44,6 +43,7 @@ using Microsoft.IdentityModel.Protocols;
using Microsoft.IdentityModel.Protocols.OpenIdConnect;
using StellaOps.Concelier.WebService.Diagnostics;
using Microsoft.IdentityModel.Tokens;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.WebService.Tests;
@@ -411,10 +411,11 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
tenant: "tenant-verify-violations",
vendor: "osv",
upstreamId: "GHSA-VERIFY-ERR",
contentHash: string.Empty,
contentHash: "sha256:verify-err",
raw: new BsonDocument
{
{ "id", "GHSA-VERIFY-ERR" }
{ "id", "GHSA-VERIFY-ERR" },
{ "severity", "critical" }
}));
using var client = _factory.CreateClient();
@@ -1492,16 +1493,16 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
}
private static readonly DateTimeOffset DefaultIngestTimestamp = new(2025, 1, 1, 0, 0, 0, TimeSpan.Zero);
private static readonly ICryptoHash Hash = CryptoHashFactory.CreateDefault();
private static string ComputeContentHash(BsonDocument rawDocument)
{
using var sha256 = SHA256.Create();
var canonical = rawDocument.ToJson(new JsonWriterSettings
{
OutputMode = JsonOutputMode.RelaxedExtendedJson
});
var bytes = sha256.ComputeHash(Encoding.UTF8.GetBytes(canonical));
return $"sha256:{Convert.ToHexString(bytes).ToLowerInvariant()}";
var digest = Hash.ComputeHashHex(Encoding.UTF8.GetBytes(canonical), HashAlgorithms.Sha256);
return $"sha256:{digest}";
}
private static string ComputeDeterministicContentHash(string upstreamId)
@@ -1522,9 +1523,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
return value.Trim();
}
using var sha256 = SHA256.Create();
var bytes = sha256.ComputeHash(Encoding.UTF8.GetBytes(raw.GetRawText()));
return $"sha256:{Convert.ToHexString(bytes).ToLowerInvariant()}";
var digest = Hash.ComputeHashHex(Encoding.UTF8.GetBytes(raw.GetRawText()), HashAlgorithms.Sha256);
return $"sha256:{digest}";
}
private sealed record ReplayResponse(