up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Policy Simulation / policy-simulate (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-25 22:09:44 +02:00
parent 6bee1fdcf5
commit 9f6e6f7fb3
116 changed files with 4495 additions and 730 deletions

View File

@@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
using StellaOps.Concelier.Core.Attestation;
namespace StellaOps.Concelier.WebService;

View File

@@ -19,8 +19,6 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Hosting;
using System.Diagnostics;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Primitives;
@@ -32,11 +30,11 @@ using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.WebService.Diagnostics;
using ServiceStatus = StellaOps.Concelier.WebService.Diagnostics.ServiceStatus;
using Serilog;
using StellaOps.Concelier.Merge;
using StellaOps.Concelier.Merge.Services;
using StellaOps.Concelier.WebService.Extensions;
using StellaOps.Concelier.WebService.Services;
using StellaOps.Concelier.WebService.Jobs;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.WebService.Filters;
@@ -61,8 +59,8 @@ using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Aliases;
using StellaOps.Provenance.Mongo;
using StellaOps.Concelier.Core.Attestation;
using AttestationClaims = StellaOps.Concelier.Core.Attestation.AttestationClaims;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
using System.Security.Cryptography;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Models.Observations;
@@ -117,28 +115,33 @@ var contentRootPath = builder.Environment.ContentRootPath;
ConcelierOptions concelierOptions;
if (builder.Environment.IsEnvironment("Testing"))
{
// Allow a fully pre-bound options instance to be supplied by the test host.
#pragma warning disable ASP0000 // test-only: create provider to fetch pre-bound options
using var tempProvider = builder.Services.BuildServiceProvider();
#pragma warning restore ASP0000
concelierOptions = tempProvider.GetService<IOptions<ConcelierOptions>>()?.Value ?? new ConcelierOptions
{
Storage = new ConcelierOptions.StorageOptions
// Allow a fully pre-bound options instance to be supplied by the test host.
#pragma warning disable ASP0000 // test-only: create provider to fetch pre-bound options
using var tempProvider = builder.Services.BuildServiceProvider();
#pragma warning restore ASP0000
concelierOptions = tempProvider.GetService<IOptions<ConcelierOptions>>()?.Value ?? new ConcelierOptions
{
Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/test-health",
Driver = "mongo",
CommandTimeoutSeconds = 30
},
Telemetry = new ConcelierOptions.TelemetryOptions
{
Enabled = false
}
};
Storage = new ConcelierOptions.StorageOptions
{
Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/test-health",
Driver = "mongo",
CommandTimeoutSeconds = 30
},
Telemetry = new ConcelierOptions.TelemetryOptions
{
Enabled = false
}
};
ConcelierOptionsPostConfigure.Apply(concelierOptions, contentRootPath);
// Skip validation in Testing to allow factory-provided wiring.
}
concelierOptions.Storage ??= new ConcelierOptions.StorageOptions();
concelierOptions.Storage.Dsn = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN") ?? "mongodb://localhost:27017/orch-tests";
concelierOptions.Storage.Driver = "mongo";
concelierOptions.Storage.CommandTimeoutSeconds = concelierOptions.Storage.CommandTimeoutSeconds <= 0 ? 30 : concelierOptions.Storage.CommandTimeoutSeconds;
ConcelierOptionsPostConfigure.Apply(concelierOptions, contentRootPath);
// Skip validation in Testing to allow factory-provided wiring.
}
else
{
concelierOptions = builder.Configuration.BindOptions<ConcelierOptions>(postConfigure: (opts, _) =>
@@ -171,12 +174,27 @@ builder.Services.AddMemoryCache();
builder.Services.AddSingleton<MirrorRateLimiter>();
builder.Services.AddSingleton<MirrorFileLocator>();
builder.Services.AddMongoStorage(storageOptions =>
var isTesting = builder.Environment.IsEnvironment("Testing");
var mongoBypass = isTesting || string.Equals(
Environment.GetEnvironmentVariable("CONCELIER_BYPASS_MONGO"),
"1",
StringComparison.OrdinalIgnoreCase);
if (!isTesting)
{
storageOptions.ConnectionString = concelierOptions.Storage.Dsn;
storageOptions.DatabaseName = concelierOptions.Storage.Database;
storageOptions.CommandTimeout = TimeSpan.FromSeconds(concelierOptions.Storage.CommandTimeoutSeconds);
});
builder.Services.AddMongoStorage(storageOptions =>
{
storageOptions.ConnectionString = concelierOptions.Storage.Dsn;
storageOptions.DatabaseName = concelierOptions.Storage.Database;
storageOptions.CommandTimeout = TimeSpan.FromSeconds(concelierOptions.Storage.CommandTimeoutSeconds);
});
}
else
{
// In test host we entirely bypass Mongo validation/bootstrapping; tests inject fakes.
builder.Services.RemoveAll<IMongoClient>();
builder.Services.RemoveAll<IMongoDatabase>();
}
builder.Services.AddOptions<AdvisoryObservationEventPublisherOptions>()
.Bind(builder.Configuration.GetSection("advisoryObservationEvents"))
.PostConfigure(options =>
@@ -229,7 +247,7 @@ builder.Services.PostConfigure<JobSchedulerOptions>(options =>
});
builder.Services.AddSingleton<OpenApiDiscoveryDocumentProvider>();
builder.Services.AddSingleton<ServiceStatus>(sp => new ServiceStatus(sp.GetRequiredService<TimeProvider>()));
builder.Services.AddSingleton<StellaOps.Concelier.WebService.Diagnostics.ServiceStatus>(sp => new StellaOps.Concelier.WebService.Diagnostics.ServiceStatus(sp.GetRequiredService<TimeProvider>()));
builder.Services.AddAocGuard();
var authorityConfigured = concelierOptions.Authority is { Enabled: true };
@@ -743,7 +761,7 @@ app.MapGet("/v1/lnm/linksets", async (
resolvedPageSize,
cancellationToken).ConfigureAwait(false);
var items = new List<LnmLinksetResponse>(result.Items.Length);
var items = new List<LnmLinksetResponse>(result.Items.Count);
foreach (var linkset in result.Items)
{
var summary = await BuildObservationSummaryAsync(observationQueryService, tenant!, linkset, cancellationToken).ConfigureAwait(false);
@@ -788,7 +806,7 @@ app.MapPost("/v1/lnm/linksets/search", async (
resolvedPageSize,
cancellationToken).ConfigureAwait(false);
var items = new List<LnmLinksetResponse>(result.Items.Length);
var items = new List<LnmLinksetResponse>(result.Items.Count);
foreach (var linkset in result.Items)
{
var summary = await BuildObservationSummaryAsync(observationQueryService, tenant!, linkset, cancellationToken).ConfigureAwait(false);
@@ -1346,7 +1364,7 @@ var evidenceSnapshotEndpoint = app.MapGet("/obs/evidence/advisories/{advisoryKey
var hash = await ComputeSha256Async(manifestStream, cancellationToken).ConfigureAwait(false);
var response = new EvidenceSnapshotResponse(
advisoryKey: advisoryKey.Trim(),
AdvisoryKey: advisoryKey.Trim(),
Tenant: tenant,
ManifestPath: manifestPath,
ManifestHash: hash,
@@ -2241,33 +2259,8 @@ async Task<LinksetObservationSummary> BuildObservationSummaryAsync(
return LinksetObservationSummary.Empty;
}
var published = result.Observations
.Where(o => o.Published.HasValue)
.Select(o => o.Published!.Value)
.OrderBy(p => p)
.FirstOrDefault();
var modified = result.Observations
.Where(o => o.Modified.HasValue)
.Select(o => o.Modified!.Value)
.OrderByDescending(p => p)
.FirstOrDefault();
var severity = result.Observations
.SelectMany(o => o.Severities)
.OrderByDescending(s => s.Score)
.FirstOrDefault();
var severityText = severity is null ? null : $"{severity.System}:{severity.Score:0.0}";
var evidenceHash = result.Observations
.Select(o => o.Provenance.SourceArtifactSha)
.FirstOrDefault();
return new LinksetObservationSummary(
PublishedAt: published == default ? null : published,
ModifiedAt: modified == default ? null : modified,
Severity: severityText,
EvidenceHash: evidenceHash);
// Observation timelines are not yet populated; return empty summary until ingestion enriches these fields.
return LinksetObservationSummary.Empty;
}
IReadOnlyList<LnmLinksetTimeline> BuildTimeline(AdvisoryLinkset linkset, LinksetObservationSummary summary)
@@ -2290,15 +2283,6 @@ IReadOnlyList<LnmLinksetTimeline> BuildTimeline(AdvisoryLinkset linkset, Linkset
return timeline;
}
readonly record struct LinksetObservationSummary(
DateTimeOffset? PublishedAt,
DateTimeOffset? ModifiedAt,
string? Severity,
string? EvidenceHash)
{
public static LinksetObservationSummary Empty { get; } = new(null, null, null, null);
}
IResult JsonResult<T>(T value, int? statusCode = null)
{
var payload = JsonSerializer.Serialize(value, JsonOptions);
@@ -2834,7 +2818,7 @@ void ApplyNoCache(HttpResponse response)
await InitializeMongoAsync(app);
app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServices] ServiceStatus status, HttpContext context) =>
app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status, HttpContext context) =>
{
ApplyNoCache(context.Response);
@@ -2863,7 +2847,7 @@ app.MapGet("/health", ([FromServices] IOptions<ConcelierOptions> opts, [FromServ
return JsonResult(response);
});
app.MapGet("/ready", async ([FromServices] IMongoDatabase database, [FromServices] ServiceStatus status, HttpContext context, CancellationToken cancellationToken) =>
app.MapGet("/ready", async ([FromServices] IMongoDatabase database, [FromServices] StellaOps.Concelier.WebService.Diagnostics.ServiceStatus status, HttpContext context, CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -3252,13 +3236,23 @@ var concelierTimelineEndpoint = app.MapGet("/obs/concelier/timeline", async (
await app.RunAsync();
}
static JsonSerializerOptions CreateJsonOptions()
static JsonSerializerOptions CreateJsonOptions()
{
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web);
options.Converters.Add(new JsonStringEnumConverter());
return options;
}
// Linkset summary used by advisory summary timeline
private readonly record struct LinksetObservationSummary(
DateTimeOffset? PublishedAt,
DateTimeOffset? ModifiedAt,
string? Severity,
string? EvidenceHash)
{
public static LinksetObservationSummary Empty { get; } = new(null, null, null, null);
}
static PluginHostOptions BuildPluginOptions(ConcelierOptions options, string contentRoot)
{
var pluginOptions = new PluginHostOptions
@@ -3293,7 +3287,7 @@ static async Task InitializeMongoAsync(WebApplication app)
await using var scope = app.Services.CreateAsyncScope();
var bootstrapper = scope.ServiceProvider.GetRequiredService<MongoBootstrapper>();
var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("MongoBootstrapper");
var status = scope.ServiceProvider.GetRequiredService<ServiceStatus>();
var status = scope.ServiceProvider.GetRequiredService<StellaOps.Concelier.WebService.Diagnostics.ServiceStatus>();
var stopwatch = Stopwatch.StartNew();

View File

@@ -1,4 +1,5 @@
using System.Text.Json;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Services;

View File

@@ -66,12 +66,36 @@ public sealed class MongoStorageOptions
return MongoStorageDefaults.DefaultDatabaseName;
}
public void EnsureValid()
{
if (string.IsNullOrWhiteSpace(ConnectionString))
{
throw new InvalidOperationException("Mongo connection string is not configured.");
}
public void EnsureValid()
{
var isTesting = string.Equals(
Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT"),
"Testing",
StringComparison.OrdinalIgnoreCase);
var bypass = string.Equals(
Environment.GetEnvironmentVariable("CONCELIER_BYPASS_MONGO"),
"1",
StringComparison.OrdinalIgnoreCase);
if (isTesting || bypass)
{
// Under test, skip validation entirely; callers may stub Mongo.
return;
}
if (string.IsNullOrWhiteSpace(ConnectionString))
{
var fallback = Environment.GetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN");
if (!string.IsNullOrWhiteSpace(fallback))
{
ConnectionString = fallback;
}
}
if (string.IsNullOrWhiteSpace(ConnectionString))
{
throw new InvalidOperationException("Mongo connection string is not configured.");
}
if (CommandTimeout <= TimeSpan.Zero)
{

View File

@@ -29,14 +29,32 @@ namespace StellaOps.Concelier.Storage.Mongo;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddMongoStorage(this IServiceCollection services, Action<MongoStorageOptions> configureOptions)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configureOptions);
services.AddOptions<MongoStorageOptions>()
.Configure(configureOptions)
.PostConfigure(static options => options.EnsureValid());
public static IServiceCollection AddMongoStorage(this IServiceCollection services, Action<MongoStorageOptions> configureOptions)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configureOptions);
// In unit/integration tests we bypass Mongo wiring entirely; callers may inject fakes.
var isTesting = string.Equals(
Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT"),
"Testing",
StringComparison.OrdinalIgnoreCase);
var bypass = string.Equals(
Environment.GetEnvironmentVariable("CONCELIER_BYPASS_MONGO"),
"1",
StringComparison.OrdinalIgnoreCase);
if (isTesting || bypass)
{
return services;
}
services.AddOptions<MongoStorageOptions>()
.Configure(configureOptions)
.PostConfigure(static options =>
{
// Normal path: enforce validity.
options.EnsureValid();
});
services.TryAddSingleton(TimeProvider.System);

View File

@@ -0,0 +1,196 @@
using System;
using System.Net;
using System.Collections.Generic;
using System.Net.Http.Json;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
using StellaOps.Concelier.WebService;
using StellaOps.Concelier.WebService.Options;
using Xunit;
namespace StellaOps.Concelier.WebService.Tests;
public sealed class OrchestratorTestWebAppFactory : WebApplicationFactory<Program>
{
public OrchestratorTestWebAppFactory()
{
Environment.SetEnvironmentVariable("CONCELIER__STORAGE__DSN", "mongodb://localhost:27017/orch-tests");
Environment.SetEnvironmentVariable("CONCELIER__STORAGE__DRIVER", "mongo");
Environment.SetEnvironmentVariable("CONCELIER__STORAGE__COMMANDTIMEOUTSECONDS", "30");
Environment.SetEnvironmentVariable("CONCELIER__TELEMETRY__ENABLED", "false");
Environment.SetEnvironmentVariable("CONCELIER__AUTHORITY__ENABLED", "false"); // disable auth so tests can hit endpoints without tokens
Environment.SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", "1");
Environment.SetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN", "mongodb://localhost:27017/orch-tests");
Environment.SetEnvironmentVariable("CONCELIER_BYPASS_MONGO", "1");
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Testing");
Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Testing");
}
protected override void ConfigureWebHost(IWebHostBuilder builder)
{
builder.UseEnvironment("Testing");
builder.ConfigureAppConfiguration(cfg =>
{
cfg.AddInMemoryCollection(new Dictionary<string, string?>
{
["Concelier:Storage:Dsn"] = "mongodb://localhost:27017/orch-tests",
["Concelier:Storage:Driver"] = "mongo",
["Concelier:Storage:CommandTimeoutSeconds"] = "30",
["Concelier:Telemetry:Enabled"] = "false",
["Concelier:Authority:Enabled"] = "false"
});
});
builder.ConfigureServices(services =>
{
services.RemoveAll<IOrchestratorRegistryStore>();
services.AddSingleton<IOrchestratorRegistryStore, InMemoryOrchestratorStore>();
// Pre-bind options to keep Program from trying to rebind/validate during tests.
services.RemoveAll<ConcelierOptions>();
services.RemoveAll<IOptions<ConcelierOptions>>();
var forcedOptions = new ConcelierOptions
{
Storage = new ConcelierOptions.StorageOptions
{
Dsn = "mongodb://localhost:27017/orch-tests",
Driver = "mongo",
CommandTimeoutSeconds = 30
},
Telemetry = new ConcelierOptions.TelemetryOptions
{
Enabled = false
},
Authority = new ConcelierOptions.AuthorityOptions
{
Enabled = false
}
};
services.AddSingleton(forcedOptions);
services.AddSingleton<IOptions<ConcelierOptions>>(_ => Microsoft.Extensions.Options.Options.Create(forcedOptions));
// Force Mongo storage options to a deterministic in-memory test DSN.
services.PostConfigure<MongoStorageOptions>(opts =>
{
opts.ConnectionString = "mongodb://localhost:27017/orch-tests";
opts.DatabaseName = "orch-tests";
opts.CommandTimeout = TimeSpan.FromSeconds(30);
});
});
}
}
public sealed class OrchestratorEndpointsTests : IClassFixture<OrchestratorTestWebAppFactory>
{
private readonly OrchestratorTestWebAppFactory _factory;
public OrchestratorEndpointsTests(OrchestratorTestWebAppFactory factory) => _factory = factory;
[Fact]
public async Task Registry_accepts_valid_request_with_tenant()
{
var client = _factory.CreateClient();
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tenant-a");
var request = new
{
connectorId = "demo-connector",
source = "demo-source",
capabilities = new[] { "ingest" },
authRef = "secret",
schedule = new { cron = "0 0 * * *", timeZone = "UTC", maxParallelRuns = 1, maxLagMinutes = 5 },
ratePolicy = new { rpm = 100, burst = 10, cooldownSeconds = 5 },
artifactKinds = new[] { "advisory" },
lockKey = "lk-1",
egressGuard = new { allowlist = new[] { "example.com" }, airgapMode = false }
};
var response = await client.PostAsJsonAsync("/internal/orch/registry", request);
response.StatusCode.Should().Be(HttpStatusCode.Accepted);
}
[Fact]
public async Task Heartbeat_accepts_valid_request_with_tenant()
{
var client = _factory.CreateClient();
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tenant-a");
var request = new
{
connectorId = "demo-connector",
runId = "11111111-1111-1111-1111-111111111111",
sequence = 1,
status = "running",
progress = 10,
queueDepth = 0
};
var response = await client.PostAsJsonAsync("/internal/orch/heartbeat", request);
response.StatusCode.Should().Be(HttpStatusCode.Accepted);
}
[Fact]
public async Task Commands_get_returns_ok_with_empty_list()
{
var client = _factory.CreateClient();
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tenant-a");
var response = await client.GetAsync("/internal/orch/commands?connectorId=demo-connector&runId=11111111-1111-1111-1111-111111111111");
response.StatusCode.Should().Be(HttpStatusCode.OK);
var payload = await response.Content.ReadFromJsonAsync<List<OrchestratorCommandRecord>>();
payload.Should().NotBeNull();
payload!.Should().BeEmpty();
}
}
internal sealed class InMemoryOrchestratorStore : IOrchestratorRegistryStore
{
private readonly Dictionary<(string Tenant, string ConnectorId), OrchestratorRegistryRecord> _registry = new();
private readonly List<OrchestratorHeartbeatRecord> _heartbeats = new();
private readonly List<OrchestratorCommandRecord> _commands = new();
public Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken)
{
_registry[(record.Tenant, record.ConnectorId)] = record;
return Task.CompletedTask;
}
public Task<OrchestratorRegistryRecord?> GetAsync(string tenant, string connectorId, CancellationToken cancellationToken)
{
_registry.TryGetValue((tenant, connectorId), out var record);
return Task.FromResult(record);
}
public Task EnqueueCommandAsync(OrchestratorCommandRecord command, CancellationToken cancellationToken)
{
_commands.Add(command);
return Task.CompletedTask;
}
public Task<IReadOnlyList<OrchestratorCommandRecord>> GetPendingCommandsAsync(string tenant, string connectorId, Guid runId, long? afterSequence, CancellationToken cancellationToken)
{
var items = _commands
.Where(c => c.Tenant == tenant && c.ConnectorId == connectorId && c.RunId == runId && (afterSequence is null || c.Sequence > afterSequence))
.ToList()
.AsReadOnly();
return Task.FromResult<IReadOnlyList<OrchestratorCommandRecord>>(items);
}
public Task AppendHeartbeatAsync(OrchestratorHeartbeatRecord heartbeat, CancellationToken cancellationToken)
{
_heartbeats.Add(heartbeat);
return Task.CompletedTask;
}
}

View File

@@ -4,6 +4,7 @@ using System.Threading.Tasks;
using FluentAssertions;
using StellaOps.Concelier.WebService.Services;
using StellaOps.Concelier.WebService;
using StellaOps.Concelier.WebService.Options;
using Xunit;
namespace StellaOps.Concelier.WebService.Tests.Services;