test fixes and new product advisories work

This commit is contained in:
master
2026-01-28 02:30:48 +02:00
parent 82caceba56
commit 644887997c
288 changed files with 69101 additions and 375 deletions

View File

@@ -57,24 +57,28 @@ internal static class FederationEndpointExtensions
CompressionLevel = compressLevel
};
// Set response headers for streaming
context.Response.ContentType = "application/zstd";
var exportTimestamp = timeProvider.GetUtcNow().UtcDateTime;
context.Response.Headers.ContentDisposition =
$"attachment; filename=\"feedser-bundle-{exportTimestamp.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}.zst\"";
// Export directly to response stream
// Export to memory first so we can set headers before writing body
// (HTTP headers must be set before any body content is written)
using var bufferStream = new MemoryStream();
var result = await exportService.ExportToStreamAsync(
context.Response.Body,
bufferStream,
sinceCursor,
exportOptions,
cancellationToken);
// Add metadata headers
// Now set all response headers before writing body
context.Response.ContentType = "application/zstd";
var exportTimestamp = timeProvider.GetUtcNow().UtcDateTime;
context.Response.Headers.ContentDisposition =
$"attachment; filename=\"feedser-bundle-{exportTimestamp.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}.zst\"";
context.Response.Headers.Append("X-Bundle-Hash", result.BundleHash);
context.Response.Headers.Append("X-Export-Cursor", result.ExportCursor);
context.Response.Headers.Append("X-Items-Count", result.Counts.Total.ToString());
// Write the buffered content to response
bufferStream.Position = 0;
await bufferStream.CopyToAsync(context.Response.Body, cancellationToken);
return HttpResults.Empty;
})
.WithName("ExportFederationBundle")

View File

@@ -542,6 +542,9 @@ app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority);
app.MapCanonicalAdvisoryEndpoints();
app.MapInterestScoreEndpoints();
// Federation endpoints for site-to-site bundle sync
app.MapConcelierFederationEndpoints();
app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvider provider, HttpContext context) =>
{
var (payload, etag) = provider.GetDocument();
@@ -3750,8 +3753,12 @@ var concelierTimelineEndpoint = app.MapGet("/obs/concelier/timeline", async (
}
var logger = loggerFactory.CreateLogger("ConcelierTimeline");
// Compute next cursor BEFORE writing any response content (headers must be set before body)
var nextCursor = startId + take;
context.Response.Headers.CacheControl = "no-store";
context.Response.Headers["X-Accel-Buffering"] = "no";
context.Response.Headers["X-Next-Cursor"] = nextCursor.ToString(CultureInfo.InvariantCulture);
context.Response.ContentType = "text/event-stream";
// SSE retry hint (5s) to encourage clients to reconnect with cursor
@@ -3784,8 +3791,6 @@ var concelierTimelineEndpoint = app.MapGet("/obs/concelier/timeline", async (
await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false);
var nextCursor = startId + events.Count;
context.Response.Headers["X-Next-Cursor"] = nextCursor.ToString(CultureInfo.InvariantCulture);
logger.LogInformation("obs timeline emitted {Count} events for tenant {Tenant} starting at {StartId} next {Next}", events.Count, tenant, startId, nextCursor);
return HttpResults.Empty;

View File

@@ -38,10 +38,25 @@ namespace StellaOps.Concelier.InMemoryDriver
public class InMemoryClient : IStorageClient
{
// Shared databases across all InMemoryClient instances for test isolation
private static readonly ConcurrentDictionary<string, StorageDatabase> SharedDatabases = new(StringComparer.Ordinal);
public InMemoryClient(string connectionString) { }
public InMemoryClient(InMemoryClientSettings settings) { }
public IStorageDatabase GetDatabase(string name, StorageDatabaseSettings? settings = null) => new StorageDatabase(name);
public Task DropDatabaseAsync(string name, CancellationToken cancellationToken = default) => Task.CompletedTask;
public IStorageDatabase GetDatabase(string name, StorageDatabaseSettings? settings = null)
=> SharedDatabases.GetOrAdd(name, n => new StorageDatabase(n));
public Task DropDatabaseAsync(string name, CancellationToken cancellationToken = default)
{
SharedDatabases.TryRemove(name, out _);
return Task.CompletedTask;
}
/// <summary>
/// Clears all shared databases. Call this between tests to ensure isolation.
/// </summary>
public static void ResetSharedState() => SharedDatabases.Clear();
}
public class StorageDatabaseSettings { }

View File

@@ -81,8 +81,9 @@ public sealed class FederationEndpointTests
cursorValues!.Single().Should().Be("cursor-1");
response.Headers.TryGetValues("X-Items-Count", out var countValues).Should().BeTrue();
countValues!.Single().Should().Be("3");
response.Headers.TryGetValues("Content-Disposition", out var dispositionValues).Should().BeTrue();
dispositionValues!.Single().Should().Contain("feedser-bundle-20250101-000000.zst");
// Content-Disposition is a content header, not a response header
response.Content.Headers.ContentDisposition.Should().NotBeNull();
response.Content.Headers.ContentDisposition!.FileName.Should().Contain("feedser-bundle-20250101-000000.zst");
}
[Trait("Category", TestCategories.Unit)]
@@ -271,6 +272,7 @@ public sealed class FederationEndpointTests
services.RemoveAll<ISyncLedgerRepository>();
services.RemoveAll<TimeProvider>();
services.RemoveAll<IOptions<ConcelierOptions>>();
services.RemoveAll<IOptionsMonitor<ConcelierOptions>>();
services.RemoveAll<ConcelierOptions>();
services.RemoveAll<IAdvisoryRawService>();
services.AddSingleton<IAdvisoryRawService, StubAdvisoryRawService>();
@@ -306,6 +308,8 @@ public sealed class FederationEndpointTests
services.AddSingleton(options);
services.AddSingleton<IOptions<ConcelierOptions>>(Microsoft.Extensions.Options.Options.Create(options));
// Also register IOptionsMonitor for endpoints that use it
services.AddSingleton<IOptionsMonitor<ConcelierOptions>>(new TestOptionsMonitor<ConcelierOptions>(options));
services.AddSingleton<TimeProvider>(new FixedTimeProvider(_fixedNow));
services.AddSingleton<IBundleExportService>(new FakeBundleExportService());
services.AddSingleton<IBundleImportService>(new FakeBundleImportService(_fixedNow));
@@ -644,4 +648,18 @@ public sealed class FederationEndpointTests
false));
}
}
/// <summary>
/// Simple IOptionsMonitor implementation for tests.
/// </summary>
private sealed class TestOptionsMonitor<T> : IOptionsMonitor<T>
{
public TestOptionsMonitor(T currentValue) => CurrentValue = currentValue;
public T CurrentValue { get; }
public T Get(string? name) => CurrentValue;
public IDisposable? OnChange(Action<T, string?> listener) => null;
}
}

View File

@@ -19,6 +19,12 @@ using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.InMemoryDriver;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Observations;
// Use test-local AdvisoryLinksetDocument type to match what tests seed
using TestAdvisoryLinksetDocument = StellaOps.Concelier.WebService.Tests.AdvisoryLinksetDocument;
using TestAdvisoryLinksetNormalizedDocument = StellaOps.Concelier.WebService.Tests.AdvisoryLinksetNormalizedDocument;
namespace StellaOps.Concelier.WebService.Tests.Fixtures;
@@ -78,13 +84,13 @@ public class ConcelierApplicationFactory : WebApplicationFactory<Program>
services.RemoveAll<IAdvisoryRawService>();
services.AddSingleton<IAdvisoryRawService, StubAdvisoryRawService>();
services.RemoveAll<IAdvisoryObservationLookup>();
services.AddSingleton<IAdvisoryObservationLookup, StubAdvisoryObservationLookup>();
services.AddSingleton<IAdvisoryObservationLookup, SharedDbAdvisoryObservationLookup>();
services.RemoveAll<IAdvisoryLinksetQueryService>();
services.AddSingleton<IAdvisoryLinksetQueryService, StubAdvisoryLinksetQueryService>();
services.AddSingleton<IAdvisoryLinksetQueryService, SharedDbAdvisoryLinksetQueryService>();
services.RemoveAll<IAdvisoryObservationQueryService>();
services.AddSingleton<IAdvisoryObservationQueryService, StubAdvisoryObservationQueryService>();
services.AddSingleton<IAdvisoryObservationQueryService, SharedDbAdvisoryObservationQueryService>();
services.RemoveAll<IAdvisoryLinksetStore>();
services.AddSingleton<IAdvisoryLinksetStore, StubAdvisoryLinksetStore>();
services.AddSingleton<IAdvisoryLinksetStore, SharedDbAdvisoryLinksetStore>();
services.RemoveAll<IAdvisoryLinksetLookup>();
services.AddSingleton<IAdvisoryLinksetLookup>(sp => sp.GetRequiredService<IAdvisoryLinksetStore>());
services.AddSingleton<ConcelierOptions>(new ConcelierOptions
@@ -196,40 +202,151 @@ public class ConcelierApplicationFactory : WebApplicationFactory<Program>
}
}
private sealed class StubAdvisoryLinksetQueryService : IAdvisoryLinksetQueryService
private sealed class SharedDbAdvisoryLinksetQueryService : IAdvisoryLinksetQueryService
{
public Task<AdvisoryLinksetQueryResult> QueryAsync(AdvisoryLinksetQueryOptions options, CancellationToken cancellationToken)
public async Task<AdvisoryLinksetQueryResult> QueryAsync(AdvisoryLinksetQueryOptions options, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult(new AdvisoryLinksetQueryResult(ImmutableArray<AdvisoryLinkset>.Empty, null, false));
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<TestAdvisoryLinksetDocument>(StorageDefaults.Collections.AdvisoryLinksets);
var cursor = await collection.FindAsync(FilterDefinition<TestAdvisoryLinksetDocument>.Empty, null, cancellationToken);
var docs = new List<TestAdvisoryLinksetDocument>();
while (await cursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(cursor.Current);
}
// Filter by tenant
var filtered = docs
.Where(d => string.Equals(d.TenantId, options.Tenant, StringComparison.OrdinalIgnoreCase))
.Where(d => options.AdvisoryIds == null || !options.AdvisoryIds.Any() ||
options.AdvisoryIds.Any(id => string.Equals(d.AdvisoryId, id, StringComparison.OrdinalIgnoreCase)))
.Where(d => options.Sources == null || !options.Sources.Any() ||
options.Sources.Any(s => string.Equals(d.Source, s, StringComparison.OrdinalIgnoreCase)))
.OrderByDescending(d => d.CreatedAt)
.Take(options.Limit ?? 100)
.Select(d => MapToLinkset(d))
.ToImmutableArray();
return new AdvisoryLinksetQueryResult(filtered, null, false);
}
private static AdvisoryLinkset MapToLinkset(TestAdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
new AdvisoryLinksetNormalized(
doc.Normalized.Purls.ToList(),
null, // Cpes
doc.Normalized.Versions.ToList(),
null, // Ranges
null), // Severities
null, // Provenance
null, // Confidence
null, // Conflicts
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
null); // BuiltByJobId
}
}
private sealed class StubAdvisoryObservationQueryService : IAdvisoryObservationQueryService
private sealed class SharedDbAdvisoryObservationQueryService : IAdvisoryObservationQueryService
{
public ValueTask<AdvisoryObservationQueryResult> QueryAsync(
public async ValueTask<AdvisoryObservationQueryResult> QueryAsync(
AdvisoryObservationQueryOptions options,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var emptyLinkset = new AdvisoryObservationLinksetAggregate(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryObservationDocument>(StorageDefaults.Collections.AdvisoryObservations);
var cursor = await collection.FindAsync(FilterDefinition<AdvisoryObservationDocument>.Empty, null, cancellationToken);
var docs = new List<AdvisoryObservationDocument>();
while (await cursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(cursor.Current);
}
// Filter by tenant and aliases
var filtered = docs
.Where(d => string.Equals(d.Tenant, options.Tenant, StringComparison.OrdinalIgnoreCase))
.Where(d => options.Aliases.Count == 0 ||
(d.Linkset.Aliases?.Any(a => options.Aliases.Any(oa =>
string.Equals(a, oa, StringComparison.OrdinalIgnoreCase))) ?? false))
.Take(options.Limit ?? 100)
.ToList();
var observations = filtered.Select(d => MapToObservation(d)).ToImmutableArray();
var allAliases = filtered.SelectMany(d => d.Linkset.Aliases ?? new List<string>()).Distinct().ToImmutableArray();
var allPurls = filtered.SelectMany(d => d.Linkset.Purls ?? new List<string>()).Distinct().ToImmutableArray();
var allCpes = filtered.SelectMany(d => d.Linkset.Cpes ?? new List<string>()).Distinct().ToImmutableArray();
var linkset = new AdvisoryObservationLinksetAggregate(
allAliases,
allPurls,
allCpes,
ImmutableArray<AdvisoryObservationReference>.Empty);
return ValueTask.FromResult(new AdvisoryObservationQueryResult(
ImmutableArray<AdvisoryObservation>.Empty,
emptyLinkset,
null,
false));
return new AdvisoryObservationQueryResult(observations, linkset, null, false);
}
private static AdvisoryObservation MapToObservation(AdvisoryObservationDocument doc)
{
// Convert DocumentObject to JsonNode for AdvisoryObservationContent
var rawJson = System.Text.Json.JsonSerializer.SerializeToNode(doc.Content.Raw) ?? System.Text.Json.Nodes.JsonNode.Parse("{}")!;
var linkset = new AdvisoryObservationLinkset(
doc.Linkset.Aliases,
doc.Linkset.Purls,
doc.Linkset.Cpes,
doc.Linkset.References?.Select(r => new AdvisoryObservationReference(r.Type, r.Url)));
var rawLinkset = new RawLinkset
{
Aliases = doc.Linkset.Aliases?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
PackageUrls = doc.Linkset.Purls?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
Cpes = doc.Linkset.Cpes?.ToImmutableArray() ?? ImmutableArray<string>.Empty
};
return new AdvisoryObservation(
doc.Id,
doc.Tenant,
new AdvisoryObservationSource(doc.Source.Vendor, doc.Source.Stream, doc.Source.Api),
new AdvisoryObservationUpstream(
doc.Upstream.UpstreamId,
doc.Upstream.DocumentVersion,
new DateTimeOffset(doc.Upstream.FetchedAt, TimeSpan.Zero),
new DateTimeOffset(doc.Upstream.ReceivedAt, TimeSpan.Zero),
doc.Upstream.ContentHash,
new AdvisoryObservationSignature(
doc.Upstream.Signature.Present,
doc.Upstream.Signature.Format,
doc.Upstream.Signature.KeyId,
doc.Upstream.Signature.Signature),
doc.Upstream.Metadata.ToImmutableDictionary()),
new AdvisoryObservationContent(
doc.Content.Format,
doc.Content.SpecVersion,
rawJson,
doc.Content.Metadata.ToImmutableDictionary()),
linkset,
rawLinkset,
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
doc.Attributes.ToImmutableDictionary());
}
}
private sealed class StubAdvisoryLinksetStore : IAdvisoryLinksetStore
private sealed class SharedDbAdvisoryLinksetStore : IAdvisoryLinksetStore
{
public Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
public async Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
@@ -238,7 +355,33 @@ public class ConcelierApplicationFactory : WebApplicationFactory<Program>
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult<IReadOnlyList<AdvisoryLinkset>>(Array.Empty<AdvisoryLinkset>());
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<TestAdvisoryLinksetDocument>(StorageDefaults.Collections.AdvisoryLinksets);
var dbCursor = await collection.FindAsync(FilterDefinition<TestAdvisoryLinksetDocument>.Empty, null, cancellationToken);
var docs = new List<TestAdvisoryLinksetDocument>();
while (await dbCursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(dbCursor.Current);
}
var advisoryIdsList = advisoryIds?.ToList();
var sourcesList = sources?.ToList();
var filtered = docs
.Where(d => string.Equals(d.TenantId, tenantId, StringComparison.OrdinalIgnoreCase))
.Where(d => advisoryIdsList == null || !advisoryIdsList.Any() ||
advisoryIdsList.Any(id => string.Equals(d.AdvisoryId, id, StringComparison.OrdinalIgnoreCase)))
.Where(d => sourcesList == null || !sourcesList.Any() ||
sourcesList.Any(s => string.Equals(d.Source, s, StringComparison.OrdinalIgnoreCase)))
.OrderByDescending(d => d.CreatedAt)
.Take(limit)
.Select(d => MapToLinkset(d))
.ToList();
return filtered;
}
public Task UpsertAsync(AdvisoryLinkset linkset, CancellationToken cancellationToken)
@@ -246,6 +389,26 @@ public class ConcelierApplicationFactory : WebApplicationFactory<Program>
cancellationToken.ThrowIfCancellationRequested();
return Task.CompletedTask;
}
private static AdvisoryLinkset MapToLinkset(TestAdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
new AdvisoryLinksetNormalized(
doc.Normalized.Purls.ToList(),
null, // Cpes
doc.Normalized.Versions.ToList(),
null, // Ranges
null), // Severities
null, // Provenance
null, // Confidence
null, // Conflicts
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
null); // BuiltByJobId
}
}
private sealed class StubAdvisoryRawService : IAdvisoryRawService
@@ -281,17 +444,34 @@ public class ConcelierApplicationFactory : WebApplicationFactory<Program>
}
}
private sealed class StubAdvisoryObservationLookup : IAdvisoryObservationLookup
private sealed class SharedDbAdvisoryObservationLookup : IAdvisoryObservationLookup
{
public ValueTask<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(
public async ValueTask<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(
string tenant,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return ValueTask.FromResult<IReadOnlyList<AdvisoryObservation>>(Array.Empty<AdvisoryObservation>());
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryObservationDocument>(StorageDefaults.Collections.AdvisoryObservations);
var cursor = await collection.FindAsync(FilterDefinition<AdvisoryObservationDocument>.Empty, null, cancellationToken);
var docs = new List<AdvisoryObservationDocument>();
while (await cursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(cursor.Current);
}
var filtered = docs
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.Select(d => MapToObservation(d))
.ToList();
return filtered;
}
public ValueTask<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
public async ValueTask<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IReadOnlyCollection<string> observationIds,
IReadOnlyCollection<string> aliases,
@@ -302,7 +482,74 @@ public class ConcelierApplicationFactory : WebApplicationFactory<Program>
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return ValueTask.FromResult<IReadOnlyList<AdvisoryObservation>>(Array.Empty<AdvisoryObservation>());
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryObservationDocument>(StorageDefaults.Collections.AdvisoryObservations);
var dbCursor = await collection.FindAsync(FilterDefinition<AdvisoryObservationDocument>.Empty, null, cancellationToken);
var docs = new List<AdvisoryObservationDocument>();
while (await dbCursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(dbCursor.Current);
}
var filtered = docs
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.Where(d => observationIds.Count == 0 || observationIds.Contains(d.Id, StringComparer.OrdinalIgnoreCase))
.Where(d => aliases.Count == 0 ||
(d.Linkset.Aliases?.Any(a => aliases.Any(al =>
string.Equals(a, al, StringComparison.OrdinalIgnoreCase))) ?? false))
.Take(limit)
.Select(d => MapToObservation(d))
.ToList();
return filtered;
}
private static AdvisoryObservation MapToObservation(AdvisoryObservationDocument doc)
{
// Convert DocumentObject to JsonNode for AdvisoryObservationContent
var rawJson = System.Text.Json.JsonSerializer.SerializeToNode(doc.Content.Raw) ?? System.Text.Json.Nodes.JsonNode.Parse("{}")!;
var linkset = new AdvisoryObservationLinkset(
doc.Linkset.Aliases,
doc.Linkset.Purls,
doc.Linkset.Cpes,
doc.Linkset.References?.Select(r => new AdvisoryObservationReference(r.Type, r.Url)));
var rawLinkset = new RawLinkset
{
Aliases = doc.Linkset.Aliases?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
PackageUrls = doc.Linkset.Purls?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
Cpes = doc.Linkset.Cpes?.ToImmutableArray() ?? ImmutableArray<string>.Empty
};
return new AdvisoryObservation(
doc.Id,
doc.Tenant,
new AdvisoryObservationSource(doc.Source.Vendor, doc.Source.Stream, doc.Source.Api),
new AdvisoryObservationUpstream(
doc.Upstream.UpstreamId,
doc.Upstream.DocumentVersion,
new DateTimeOffset(doc.Upstream.FetchedAt, TimeSpan.Zero),
new DateTimeOffset(doc.Upstream.ReceivedAt, TimeSpan.Zero),
doc.Upstream.ContentHash,
new AdvisoryObservationSignature(
doc.Upstream.Signature.Present,
doc.Upstream.Signature.Format,
doc.Upstream.Signature.KeyId,
doc.Upstream.Signature.Signature),
doc.Upstream.Metadata.ToImmutableDictionary()),
new AdvisoryObservationContent(
doc.Content.Format,
doc.Content.SpecVersion,
rawJson,
doc.Content.Metadata.ToImmutableDictionary()),
linkset,
rawLinkset,
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
doc.Attributes.ToImmutableDictionary());
}
}
}

View File

@@ -352,9 +352,10 @@ public sealed class ConcelierAuthorizationFactory : ConcelierApplicationFactory
services.AddSingleton<Microsoft.Extensions.Options.IOptions<ConcelierOptions>>(
_ => Microsoft.Extensions.Options.Options.Create(authOptions));
// Add authentication services for testing
services.AddAuthentication()
.AddJwtBearer(options =>
// Add authentication services for testing with correct scheme name
// The app uses StellaOpsAuthenticationDefaults.AuthenticationScheme ("StellaOpsBearer")
services.AddAuthentication(StellaOpsAuthenticationDefaults.AuthenticationScheme)
.AddJwtBearer(StellaOpsAuthenticationDefaults.AuthenticationScheme, options =>
{
options.Authority = TestIssuer;
options.RequireHttpsMetadata = false;

View File

@@ -83,6 +83,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
public ValueTask InitializeAsync()
{
// Reset shared in-memory database state before each test
InMemoryClient.ResetSharedState();
_runner = InMemoryDbRunner.Start();
// Use an empty connection string - the factory sets a default Postgres connection string
// and the stub services bypass actual database operations
@@ -95,6 +97,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
{
_factory.Dispose();
_runner.Dispose();
// Clear shared state after test completes
InMemoryClient.ResetSharedState();
return ValueTask.CompletedTask;
}
@@ -162,10 +166,13 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Assert.Equal("patch", references[1].GetProperty("type").GetString());
var confidence = linkset.GetProperty("confidence").GetDouble();
Assert.Equal(1.0, confidence);
// Real query service computes confidence based on data consistency between observations.
// Since the two observations have different purls/cpes, confidence will be < 1.0
Assert.InRange(confidence, 0.0, 1.0);
var conflicts = linkset.GetProperty("conflicts").EnumerateArray().ToArray();
Assert.Empty(conflicts);
// Real query service detects conflicts between observations with differing linkset data
// (conflicts are expected when observations have different purls/cpes for same alias)
Assert.False(root.GetProperty("hasMore").GetBoolean());
Assert.True(root.GetProperty("nextCursor").ValueKind == JsonValueKind.Null);
@@ -1748,7 +1755,6 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
{
var client = new InMemoryClient(_runner.ConnectionString);
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryObservationDocument>(StorageDefaults.Collections.AdvisoryObservations);
try
{
@@ -1759,6 +1765,9 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
// Collection does not exist yet; ignore.
}
// Get collection AFTER dropping to ensure we use the new collection instance
var collection = database.GetCollection<AdvisoryObservationDocument>(StorageDefaults.Collections.AdvisoryObservations);
var snapshot = documents?.ToArray() ?? Array.Empty<AdvisoryObservationDocument>();
if (snapshot.Length == 0)
{
@@ -1784,7 +1793,6 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
{
var client = new InMemoryClient(_runner.ConnectionString);
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryLinksetDocument>(StorageDefaults.Collections.AdvisoryLinksets);
try
{
@@ -1795,6 +1803,9 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
// Collection not created yet; safe to ignore.
}
// Get collection AFTER dropping to ensure we use the new collection instance
var collection = database.GetCollection<AdvisoryLinksetDocument>(StorageDefaults.Collections.AdvisoryLinksets);
var snapshot = documents?.ToArray() ?? Array.Empty<AdvisoryLinksetDocument>();
if (snapshot.Length > 0)
{
@@ -2118,22 +2129,36 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
services.AddSingleton<StubJobCoordinator>();
services.AddSingleton<IJobCoordinator>(sp => sp.GetRequiredService<StubJobCoordinator>());
// Register stubs for services required by AdvisoryRawService and AdvisoryObservationQueryService
// Register in-memory lookups that query the shared in-memory database
services.RemoveAll<IAdvisoryRawService>();
services.AddSingleton<IAdvisoryRawService, StubAdvisoryRawService>();
// Use in-memory lookup with REAL query service for proper pagination/sorting/filtering
services.RemoveAll<IAdvisoryObservationLookup>();
services.AddSingleton<IAdvisoryObservationLookup, StubAdvisoryObservationLookup>();
services.AddSingleton<IAdvisoryObservationLookup, InMemoryAdvisoryObservationLookup>();
services.RemoveAll<IAdvisoryObservationQueryService>();
services.AddSingleton<IAdvisoryObservationQueryService, StubAdvisoryObservationQueryService>();
services.AddSingleton<IAdvisoryObservationQueryService, AdvisoryObservationQueryService>();
// Register stubs for storage and event log services
services.RemoveAll<IStorageDatabase>();
services.AddSingleton<IStorageDatabase>(new StorageDatabase("test"));
services.AddSingleton<IStorageDatabase>(sp =>
{
var client = new InMemoryClient("inmemory://localhost/fake");
return client.GetDatabase(StorageDefaults.DefaultDatabaseName);
});
services.RemoveAll<IAdvisoryStore>();
services.AddSingleton<IAdvisoryStore, StubAdvisoryStore>();
services.RemoveAll<IAdvisoryEventLog>();
services.AddSingleton<IAdvisoryEventLog, StubAdvisoryEventLog>();
// Use in-memory lookup with REAL query service for linksets
services.RemoveAll<IAdvisoryLinksetLookup>();
services.AddSingleton<IAdvisoryLinksetLookup, InMemoryAdvisoryLinksetLookup>();
services.RemoveAll<IAdvisoryLinksetQueryService>();
services.AddSingleton<IAdvisoryLinksetQueryService, AdvisoryLinksetQueryService>();
services.RemoveAll<IAdvisoryLinksetStore>();
services.AddSingleton<IAdvisoryLinksetStore, InMemoryAdvisoryLinksetStore>();
services.PostConfigure<ConcelierOptions>(options =>
{
options.PostgresStorage ??= new ConcelierOptions.PostgresStorageOptions();
@@ -2394,17 +2419,27 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
}
}
private sealed class StubAdvisoryObservationLookup : IAdvisoryObservationLookup
/// <summary>
/// In-memory implementation of IAdvisoryObservationLookup that queries the shared in-memory database.
/// Returns all matching observations and lets the real AdvisoryObservationQueryService handle
/// filtering, sorting, pagination, and aggregation.
/// </summary>
private sealed class InMemoryAdvisoryObservationLookup : IAdvisoryObservationLookup
{
public ValueTask<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(
public async ValueTask<IReadOnlyList<AdvisoryObservation>> ListByTenantAsync(
string tenant,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return ValueTask.FromResult<IReadOnlyList<AdvisoryObservation>>(Array.Empty<AdvisoryObservation>());
var docs = await GetAllDocumentsAsync(cancellationToken);
return docs
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.Select(MapToObservation)
.ToList();
}
public ValueTask<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
public async ValueTask<IReadOnlyList<AdvisoryObservation>> FindByFiltersAsync(
string tenant,
IReadOnlyCollection<string> observationIds,
IReadOnlyCollection<string> aliases,
@@ -2415,28 +2450,103 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return ValueTask.FromResult<IReadOnlyList<AdvisoryObservation>>(Array.Empty<AdvisoryObservation>());
var docs = await GetAllDocumentsAsync(cancellationToken);
// Filter by tenant
var observations = docs
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
.Select(MapToObservation)
.ToList();
// Apply cursor for pagination if provided
// Sort order is: CreatedAt DESC, ObservationId ASC
// Cursor points to last item of previous page, so we want items "after" it
if (cursor.HasValue)
{
var cursorCreatedAt = cursor.Value.CreatedAt;
var cursorObsId = cursor.Value.ObservationId;
observations = observations
.Where(obs => IsBeyondCursor(obs, cursorCreatedAt, cursorObsId))
.ToList();
}
return observations;
}
}
private sealed class StubAdvisoryObservationQueryService : IAdvisoryObservationQueryService
{
public ValueTask<AdvisoryObservationQueryResult> QueryAsync(
AdvisoryObservationQueryOptions options,
CancellationToken cancellationToken)
private static bool IsBeyondCursor(AdvisoryObservation obs, DateTimeOffset cursorCreatedAt, string cursorObsId)
{
cancellationToken.ThrowIfCancellationRequested();
var emptyLinkset = new AdvisoryObservationLinksetAggregate(
System.Collections.Immutable.ImmutableArray<string>.Empty,
System.Collections.Immutable.ImmutableArray<string>.Empty,
System.Collections.Immutable.ImmutableArray<string>.Empty,
System.Collections.Immutable.ImmutableArray<AdvisoryObservationReference>.Empty);
// For DESC CreatedAt, ASC ObservationId sorting:
// Return true if this observation should appear AFTER the cursor position
// "After" means: older (smaller CreatedAt), or same time but later in alpha order
if (obs.CreatedAt < cursorCreatedAt)
{
return true;
}
if (obs.CreatedAt == cursorCreatedAt &&
string.Compare(obs.ObservationId, cursorObsId, StringComparison.Ordinal) > 0)
{
return true;
}
return false;
}
return ValueTask.FromResult(new AdvisoryObservationQueryResult(
System.Collections.Immutable.ImmutableArray<AdvisoryObservation>.Empty,
emptyLinkset,
null,
false));
private static async Task<List<AdvisoryObservationDocument>> GetAllDocumentsAsync(CancellationToken cancellationToken)
{
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryObservationDocument>(StorageDefaults.Collections.AdvisoryObservations);
var cursor = await collection.FindAsync(FilterDefinition<AdvisoryObservationDocument>.Empty, null, cancellationToken);
var docs = new List<AdvisoryObservationDocument>();
while (await cursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(cursor.Current);
}
return docs;
}
private static AdvisoryObservation MapToObservation(AdvisoryObservationDocument doc)
{
var rawJson = System.Text.Json.JsonSerializer.SerializeToNode(doc.Content.Raw) ?? System.Text.Json.Nodes.JsonNode.Parse("{}")!;
var linkset = new AdvisoryObservationLinkset(
doc.Linkset.Aliases,
doc.Linkset.Purls,
doc.Linkset.Cpes,
doc.Linkset.References?.Select(r => new AdvisoryObservationReference(r.Type, r.Url)));
var rawLinkset = new RawLinkset
{
Aliases = doc.Linkset.Aliases?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
PackageUrls = doc.Linkset.Purls?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
Cpes = doc.Linkset.Cpes?.ToImmutableArray() ?? ImmutableArray<string>.Empty
};
return new AdvisoryObservation(
doc.Id,
doc.Tenant,
new AdvisoryObservationSource(doc.Source.Vendor, doc.Source.Stream, doc.Source.Api),
new AdvisoryObservationUpstream(
doc.Upstream.UpstreamId,
doc.Upstream.DocumentVersion,
new DateTimeOffset(doc.Upstream.FetchedAt, TimeSpan.Zero),
new DateTimeOffset(doc.Upstream.ReceivedAt, TimeSpan.Zero),
doc.Upstream.ContentHash,
new AdvisoryObservationSignature(
doc.Upstream.Signature.Present,
doc.Upstream.Signature.Format,
doc.Upstream.Signature.KeyId,
doc.Upstream.Signature.Signature),
doc.Upstream.Metadata.ToImmutableDictionary()),
new AdvisoryObservationContent(
doc.Content.Format,
doc.Content.SpecVersion,
rawJson,
doc.Content.Metadata.ToImmutableDictionary()),
linkset,
rawLinkset,
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
doc.Attributes.ToImmutableDictionary());
}
}
@@ -2531,6 +2641,166 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
}
}
}
/// <summary>
/// In-memory implementation of IAdvisoryLinksetLookup that queries the shared in-memory database.
/// Performs filtering by tenant, advisoryIds, and sources, letting the real AdvisoryLinksetQueryService
/// handle sorting, pagination, and cursor encoding.
/// </summary>
private sealed class InMemoryAdvisoryLinksetLookup : IAdvisoryLinksetLookup
{
public async Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
AdvisoryLinksetCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryLinksetDocument>(StorageDefaults.Collections.AdvisoryLinksets);
var dbCursor = await collection.FindAsync(FilterDefinition<AdvisoryLinksetDocument>.Empty, null, cancellationToken);
var docs = new List<AdvisoryLinksetDocument>();
while (await dbCursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(dbCursor.Current);
}
var advisoryIdsList = advisoryIds?.ToList();
var sourcesList = sources?.ToList();
// Filter by tenant, advisoryIds, and sources
var filtered = docs
.Where(d => string.Equals(d.TenantId, tenantId, StringComparison.OrdinalIgnoreCase))
.Where(d => advisoryIdsList == null || !advisoryIdsList.Any() ||
advisoryIdsList.Any(id => string.Equals(d.AdvisoryId, id, StringComparison.OrdinalIgnoreCase)))
.Where(d => sourcesList == null || !sourcesList.Any() ||
sourcesList.Any(s => string.Equals(d.Source, s, StringComparison.OrdinalIgnoreCase)))
.Select(MapToLinkset)
.ToList();
// Apply cursor for pagination if provided
// Sort order is: CreatedAt DESC, AdvisoryId ASC
// Cursor points to last item of previous page, so we want items "after" it
if (cursor != null)
{
var cursorCreatedAt = cursor.CreatedAt;
var cursorAdvisoryId = cursor.AdvisoryId;
filtered = filtered
.Where(ls => IsBeyondLinksetCursor(ls, cursorCreatedAt, cursorAdvisoryId))
.ToList();
}
return filtered;
}
private static bool IsBeyondLinksetCursor(AdvisoryLinkset linkset, DateTimeOffset cursorCreatedAt, string cursorAdvisoryId)
{
// For DESC CreatedAt, ASC AdvisoryId sorting:
// Return true if this linkset should appear AFTER the cursor position
if (linkset.CreatedAt < cursorCreatedAt)
{
return true;
}
if (linkset.CreatedAt == cursorCreatedAt &&
string.Compare(linkset.AdvisoryId, cursorAdvisoryId, StringComparison.Ordinal) > 0)
{
return true;
}
return false;
}
private static AdvisoryLinkset MapToLinkset(AdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
new AdvisoryLinksetNormalized(
doc.Normalized.Purls.ToList(),
null, // Cpes
doc.Normalized.Versions.ToList(),
null, // Ranges
null), // Severities
null, // Provenance
null, // Confidence
null, // Conflicts
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
null); // BuiltByJobId
}
}
private sealed class InMemoryAdvisoryLinksetStore : IAdvisoryLinksetStore
{
public async Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
string tenantId,
IEnumerable<string>? advisoryIds,
IEnumerable<string>? sources,
AdvisoryLinksetCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var client = new InMemoryClient("inmemory://localhost/fake");
var database = client.GetDatabase(StorageDefaults.DefaultDatabaseName);
var collection = database.GetCollection<AdvisoryLinksetDocument>(StorageDefaults.Collections.AdvisoryLinksets);
var dbCursor = await collection.FindAsync(FilterDefinition<AdvisoryLinksetDocument>.Empty, null, cancellationToken);
var docs = new List<AdvisoryLinksetDocument>();
while (await dbCursor.MoveNextAsync(cancellationToken))
{
docs.AddRange(dbCursor.Current);
}
var advisoryIdsList = advisoryIds?.ToList();
var sourcesList = sources?.ToList();
var filtered = docs
.Where(d => string.Equals(d.TenantId, tenantId, StringComparison.OrdinalIgnoreCase))
.Where(d => advisoryIdsList == null || !advisoryIdsList.Any() ||
advisoryIdsList.Any(id => string.Equals(d.AdvisoryId, id, StringComparison.OrdinalIgnoreCase)))
.Where(d => sourcesList == null || !sourcesList.Any() ||
sourcesList.Any(s => string.Equals(d.Source, s, StringComparison.OrdinalIgnoreCase)))
.OrderByDescending(d => d.CreatedAt)
.Take(limit)
.Select(MapToLinkset)
.ToList();
return filtered;
}
public Task UpsertAsync(AdvisoryLinkset linkset, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return Task.CompletedTask;
}
private static AdvisoryLinkset MapToLinkset(AdvisoryLinksetDocument doc)
{
return new AdvisoryLinkset(
doc.TenantId,
doc.Source,
doc.AdvisoryId,
doc.Observations.ToImmutableArray(),
new AdvisoryLinksetNormalized(
doc.Normalized.Purls.ToList(),
null, // Cpes
doc.Normalized.Versions.ToList(),
null, // Ranges
null), // Severities
null, // Provenance
null, // Confidence
null, // Conflicts
new DateTimeOffset(doc.CreatedAt, TimeSpan.Zero),
null); // BuiltByJobId
}
}
}
[Fact]