consolidation of some of the modules, localization fixes, product advisories work, qa work

This commit is contained in:
master
2026-03-05 03:54:22 +02:00
parent 7bafcc3eef
commit 8e1cb9448d
3878 changed files with 72600 additions and 46861 deletions

View File

@@ -4,10 +4,13 @@ using Microsoft.Extensions.Options;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Options;
using StellaOps.Unknowns.Core.Models;
using StellaOps.Unknowns.Core.Repositories;
using StellaOps.Unknowns.Persistence.EfCore.Repositories;
using StellaOps.Unknowns.Persistence.Postgres;
using StellaOps.Unknowns.Persistence.Postgres.Repositories;
using Testcontainers.PostgreSql;
using Xunit;
using System.Text.Json;
using StellaOps.TestKit;
@@ -21,6 +24,7 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
private UnknownsDataSource _dataSource = null!;
private PostgresUnknownRepository _repository = null!;
private UnknownEfRepository _efRepository = null!;
private const string TestTenantId = "test-tenant";
public async ValueTask InitializeAsync()
@@ -43,6 +47,10 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
_repository = new PostgresUnknownRepository(
_dataSource,
NullLogger<PostgresUnknownRepository>.Instance);
_efRepository = new UnknownEfRepository(
_dataSource,
NullLogger<UnknownEfRepository>.Instance);
}
public async ValueTask DisposeAsync()
@@ -154,7 +162,11 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
next_scheduled_rescan TIMESTAMPTZ,
last_analyzed_at TIMESTAMPTZ,
evidence_set_hash BYTEA,
graph_slice_hash BYTEA
graph_slice_hash BYTEA,
provenance_hints JSONB NOT NULL DEFAULT '[]'::jsonb,
best_hypothesis TEXT,
combined_confidence NUMERIC(4,4) CHECK (combined_confidence IS NULL OR (combined_confidence >= 0 AND combined_confidence <= 1)),
primary_suggested_action TEXT
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_unknown_one_open_per_subject
@@ -372,7 +384,185 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
// After supersede, sys_to is set, so GetById (which filters sys_to IS NULL) returns null
result.Should().BeNull();
}
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("postgres")]
[InlineData("efcore")]
public async Task AttachProvenanceHintsAsync_ShouldPersistHints_ForBothImplementations(string implementation)
{
var repository = GetRepository(implementation);
var tenantId = $"tenant-hints-{implementation}";
var created = await repository.CreateAsync(
tenantId,
UnknownSubjectType.File,
$"file:/usr/lib/{implementation}/libcrypto.so.3",
UnknownKind.UnknownEcosystem,
UnknownSeverity.High,
"""{"path":"test"}""",
null,
null,
null,
"test-user",
CancellationToken.None);
var generatedAt = new DateTimeOffset(2026, 03, 04, 12, 00, 00, TimeSpan.Zero);
var hints = CreateHints(generatedAt);
var updated = await repository.AttachProvenanceHintsAsync(
tenantId,
created.Id,
hints,
"Debian package candidate",
0.93,
"verify_build_id",
CancellationToken.None);
updated.BestHypothesis.Should().Be("Debian package candidate");
updated.CombinedConfidence.Should().BeApproximately(0.93, 0.0001);
updated.PrimarySuggestedAction.Should().Be("verify_build_id");
updated.ProvenanceHints.Should().HaveCount(2);
updated.ProvenanceHints.Select(h => h.HintId)
.Should().ContainInOrder("hint:sha256:aaa111", "hint:sha256:bbb222");
var reloaded = await repository.GetByIdAsync(tenantId, created.Id, CancellationToken.None);
reloaded.Should().NotBeNull();
reloaded!.ProvenanceHints.Should().HaveCount(2);
reloaded.BestHypothesis.Should().Be("Debian package candidate");
reloaded.CombinedConfidence.Should().BeApproximately(0.93, 0.0001);
}
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("postgres")]
[InlineData("efcore")]
public async Task GetWithHighConfidenceHintsAsync_ShouldFilterTenantAndSortDeterministically(string implementation)
{
var repository = GetRepository(implementation);
var tenantId = $"tenant-filter-{implementation}";
var otherTenantId = $"{tenantId}-other";
var now = new DateTimeOffset(2026, 03, 04, 12, 00, 00, TimeSpan.Zero);
var highA = await repository.CreateAsync(
tenantId,
UnknownSubjectType.Package,
$"pkg:npm/high-a-{implementation}@1.0.0",
UnknownKind.MissingFeed,
UnknownSeverity.High,
null, null, null, null, "test-user", CancellationToken.None);
var highB = await repository.CreateAsync(
tenantId,
UnknownSubjectType.Package,
$"pkg:npm/high-b-{implementation}@1.0.0",
UnknownKind.MissingFeed,
UnknownSeverity.High,
null, null, null, null, "test-user", CancellationToken.None);
var low = await repository.CreateAsync(
tenantId,
UnknownSubjectType.Package,
$"pkg:npm/low-{implementation}@1.0.0",
UnknownKind.MissingFeed,
UnknownSeverity.Medium,
null, null, null, null, "test-user", CancellationToken.None);
var otherTenant = await repository.CreateAsync(
otherTenantId,
UnknownSubjectType.Package,
$"pkg:npm/other-{implementation}@1.0.0",
UnknownKind.MissingFeed,
UnknownSeverity.High,
null, null, null, null, "test-user", CancellationToken.None);
var hints = CreateHints(now);
await repository.AttachProvenanceHintsAsync(tenantId, highA.Id, hints, "High A", 0.91, "verify_build_id", CancellationToken.None);
await repository.AttachProvenanceHintsAsync(tenantId, highB.Id, hints, "High B", 0.91, "verify_build_id", CancellationToken.None);
await repository.AttachProvenanceHintsAsync(tenantId, low.Id, hints, "Low", 0.72, "manual_triage", CancellationToken.None);
await repository.AttachProvenanceHintsAsync(otherTenantId, otherTenant.Id, hints, "Other", 0.99, "manual_triage", CancellationToken.None);
var filtered = await repository.GetWithHighConfidenceHintsAsync(tenantId, minConfidence: 0.8, limit: null, CancellationToken.None);
filtered.Should().HaveCount(2);
filtered.All(item => item.TenantId == tenantId).Should().BeTrue();
filtered.All(item => item.CombinedConfidence >= 0.8).Should().BeTrue();
var expectedOrder = new[] { highA.Id, highB.Id }.OrderBy(id => id).ToArray();
filtered.Select(item => item.Id).Should().ContainInOrder(expectedOrder);
var limited = await repository.GetWithHighConfidenceHintsAsync(tenantId, minConfidence: 0.8, limit: 1, CancellationToken.None);
limited.Should().HaveCount(1);
limited[0].Id.Should().Be(expectedOrder[0]);
}
private IUnknownRepository GetRepository(string implementation)
=> implementation switch
{
"postgres" => _repository,
"efcore" => _efRepository,
_ => throw new ArgumentOutOfRangeException(nameof(implementation), implementation, null)
};
private static IReadOnlyList<ProvenanceHint> CreateHints(DateTimeOffset generatedAt)
{
return
[
new ProvenanceHint
{
HintId = "hint:sha256:bbb222",
Type = ProvenanceHintType.StringTableSignature,
Confidence = 0.62,
ConfidenceLevel = HintConfidence.Medium,
Summary = "String table signature suggests distro package",
Hypothesis = "Potential distro package from string table",
Evidence = new ProvenanceEvidence
{
Raw = JsonDocument.Parse("""{"source":"strings"}""")
},
SuggestedActions =
[
new SuggestedAction
{
Action = "manual_triage",
Priority = 2,
Effort = "medium",
Description = "Inspect package metadata manually"
}
],
GeneratedAt = generatedAt,
Source = "StringAnalyzer"
},
new ProvenanceHint
{
HintId = "hint:sha256:aaa111",
Type = ProvenanceHintType.BuildIdMatch,
Confidence = 0.91,
ConfidenceLevel = HintConfidence.High,
Summary = "Build-ID catalog match",
Hypothesis = "Debian package candidate",
Evidence = new ProvenanceEvidence
{
BuildId = new BuildIdEvidence
{
BuildId = "deadbeef",
BuildIdType = "sha256",
MatchedPackage = "openssl",
MatchedVersion = "3.0.0",
MatchedDistro = "debian",
CatalogSource = "debian-security"
}
},
SuggestedActions =
[
new SuggestedAction
{
Action = "verify_build_id",
Priority = 1,
Effort = "low",
Description = "Cross-check Build-ID with distro metadata"
}
],
GeneratedAt = generatedAt,
Source = "BuildIdAnalyzer"
}
];
}
}

View File

@@ -11,6 +11,8 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
<PackageReference Include="NSubstitute" />
<PackageReference Include="Npgsql" />
<PackageReference Include="Testcontainers.PostgreSql" />
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>

View File

@@ -0,0 +1,286 @@
using System.Net.Http.Json;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Npgsql;
using StellaOps.Unknowns.WebService.Endpoints;
using Testcontainers.PostgreSql;
using Xunit;
namespace StellaOps.Unknowns.WebService.Tests;
public sealed class UnknownsEndpointsPersistenceTests : IAsyncLifetime
{
private readonly PostgreSqlContainer _postgres = new PostgreSqlBuilder()
.WithImage("postgres:16")
.Build();
private WebApplicationFactory<Program> _factory = null!;
private HttpClient _client = null!;
private string _connectionString = string.Empty;
public async ValueTask InitializeAsync()
{
await _postgres.StartAsync();
_connectionString = _postgres.GetConnectionString();
await RunMigrationsAsync(_connectionString);
_factory = new WebApplicationFactory<Program>().WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, config) =>
{
config.AddInMemoryCollection(new Dictionary<string, string?>
{
["Postgres:ConnectionString"] = _connectionString,
["Postgres:SchemaName"] = "unknowns",
["Authority:ResourceServer:Authority"] = "http://localhost",
["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32",
["Authority:ResourceServer:BypassNetworks:1"] = "::1/128"
});
});
builder.ConfigureTestServices(UnknownsTestSecurity.Configure);
});
_client = _factory.CreateClient();
}
public async ValueTask DisposeAsync()
{
_client.Dispose();
_factory.Dispose();
await _postgres.DisposeAsync();
}
[Fact]
[Trait("Category", "Integration")]
public async Task GetHighConfidenceHints_UsesPersistenceAndTenantFiltering()
{
var tenantId = "unknowns-persist-tenant";
await SeedUnknownAsync(
tenantId,
Guid.Parse("11111111-1111-1111-1111-111111111111"),
"pkg:npm/lodash@4.17.21",
0.91m,
"Debian package candidate",
"verify_build_id");
await SeedUnknownAsync(
tenantId,
Guid.Parse("22222222-2222-2222-2222-222222222222"),
"pkg:npm/axios@0.27.2",
0.61m,
"Low confidence",
"manual_triage");
await SeedUnknownAsync(
"other-tenant",
Guid.Parse("33333333-3333-3333-3333-333333333333"),
"pkg:npm/express@4.18.2",
0.99m,
"Other tenant",
"manual_triage");
using var request = new HttpRequestMessage(
HttpMethod.Get,
"/api/unknowns/high-confidence?minConfidence=0.70&limit=10");
request.Headers.Add("X-Tenant-Id", tenantId);
var response = await _client.SendAsync(request);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadFromJsonAsync<UnknownsListResponse>();
Assert.NotNull(payload);
Assert.Single(payload.Items);
var item = payload.Items[0];
Assert.Equal(tenantId, item.TenantId);
Assert.Equal("pkg:npm/lodash@4.17.21", item.SubjectRef);
Assert.Equal("Debian package candidate", item.BestHypothesis);
Assert.Equal(0.91, item.CombinedConfidence!.Value, 3);
Assert.NotEmpty(item.ProvenanceHints);
}
private async Task SeedUnknownAsync(
string tenantId,
Guid id,
string subjectRef,
decimal combinedConfidence,
string bestHypothesis,
string suggestedAction)
{
var hintJson =
"""
[
{
"hint_id": "hint:sha256:seeded",
"type": "BuildIdMatch",
"confidence": 0.91,
"confidence_level": "High",
"summary": "Build-id matched package metadata",
"hypothesis": "Debian package candidate",
"evidence": {
"build_id": {
"build_id": "deadbeef",
"build_id_type": "sha256",
"matched_package": "openssl",
"matched_version": "3.0.0"
}
},
"suggested_actions": [
{
"action": "verify_build_id",
"priority": 1,
"effort": "low",
"description": "Verify package provenance from distro metadata"
}
],
"generated_at": "2026-03-01T10:00:00Z",
"source": "integration-seed"
}
]
""";
var subjectHash = id.ToString("N").PadRight(64, '0');
await using var dataSource = NpgsqlDataSource.Create(_connectionString);
await using var connection = await dataSource.OpenConnectionAsync();
await using var command = new NpgsqlCommand(
"""
INSERT INTO unknowns.unknown (
id, tenant_id, subject_hash, subject_type, subject_ref,
kind, severity, context, valid_from, sys_from,
created_at, created_by, updated_at,
provenance_hints, best_hypothesis, combined_confidence, primary_suggested_action
) VALUES (
@id, @tenantId, @subjectHash, 'package'::unknowns.subject_type, @subjectRef,
'missing_feed'::unknowns.unknown_kind, 'high'::unknowns.unknown_severity, '{}'::jsonb, now(), now(),
now(), 'integration-test', now(),
@hints::jsonb, @bestHypothesis, @combinedConfidence, @primarySuggestedAction
);
""",
connection);
command.Parameters.AddWithValue("id", id);
command.Parameters.AddWithValue("tenantId", tenantId);
command.Parameters.AddWithValue("subjectHash", subjectHash);
command.Parameters.AddWithValue("subjectRef", subjectRef);
command.Parameters.AddWithValue("hints", hintJson);
command.Parameters.AddWithValue("bestHypothesis", bestHypothesis);
command.Parameters.AddWithValue("combinedConfidence", combinedConfidence);
command.Parameters.AddWithValue("primarySuggestedAction", suggestedAction);
await command.ExecuteNonQueryAsync();
}
private static async Task RunMigrationsAsync(string connectionString)
{
await using var rawDataSource = NpgsqlDataSource.Create(connectionString);
await using var connection = await rawDataSource.OpenConnectionAsync();
const string schema = """
CREATE SCHEMA IF NOT EXISTS unknowns;
CREATE SCHEMA IF NOT EXISTS unknowns_app;
CREATE OR REPLACE FUNCTION unknowns_app.require_current_tenant()
RETURNS TEXT
LANGUAGE plpgsql STABLE SECURITY DEFINER
AS $$
DECLARE
v_tenant TEXT;
BEGIN
v_tenant := current_setting('app.tenant_id', true);
IF v_tenant IS NULL OR v_tenant = '' THEN
RAISE EXCEPTION 'app.tenant_id session variable not set';
END IF;
RETURN v_tenant;
END;
$$;
DO $$ BEGIN
CREATE TYPE unknowns.subject_type AS ENUM (
'package', 'ecosystem', 'version', 'sbom_edge', 'file', 'runtime'
);
EXCEPTION WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE unknowns.unknown_kind AS ENUM (
'missing_sbom', 'ambiguous_package', 'missing_feed', 'unresolved_edge',
'no_version_info', 'unknown_ecosystem', 'partial_match',
'version_range_unbounded', 'unsupported_format', 'transitive_gap'
);
EXCEPTION WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE unknowns.unknown_severity AS ENUM (
'critical', 'high', 'medium', 'low', 'info'
);
EXCEPTION WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE unknowns.resolution_type AS ENUM (
'feed_updated', 'sbom_provided', 'manual_mapping',
'superseded', 'false_positive', 'wont_fix'
);
EXCEPTION WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE unknowns.triage_band AS ENUM ('hot', 'warm', 'cold');
EXCEPTION WHEN duplicate_object THEN null;
END $$;
CREATE TABLE IF NOT EXISTS unknowns.unknown (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id TEXT NOT NULL,
subject_hash CHAR(64) NOT NULL,
subject_type unknowns.subject_type NOT NULL,
subject_ref TEXT NOT NULL,
kind unknowns.unknown_kind NOT NULL,
severity unknowns.unknown_severity,
context JSONB NOT NULL DEFAULT '{}',
source_scan_id UUID,
source_graph_id UUID,
source_sbom_digest TEXT,
valid_from TIMESTAMPTZ NOT NULL DEFAULT NOW(),
valid_to TIMESTAMPTZ,
sys_from TIMESTAMPTZ NOT NULL DEFAULT NOW(),
sys_to TIMESTAMPTZ,
resolved_at TIMESTAMPTZ,
resolution_type unknowns.resolution_type,
resolution_ref TEXT,
resolution_notes TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
created_by TEXT NOT NULL DEFAULT 'system',
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
popularity_score FLOAT DEFAULT 0.0,
deployment_count INT DEFAULT 0,
exploit_potential_score FLOAT DEFAULT 0.0,
uncertainty_score FLOAT DEFAULT 0.0,
uncertainty_flags JSONB DEFAULT '{}'::jsonb,
centrality_score FLOAT DEFAULT 0.0,
degree_centrality INT DEFAULT 0,
betweenness_centrality FLOAT DEFAULT 0.0,
staleness_score FLOAT DEFAULT 0.0,
days_since_analysis INT DEFAULT 0,
composite_score FLOAT DEFAULT 0.0,
triage_band unknowns.triage_band DEFAULT 'cold',
scoring_trace JSONB,
rescan_attempts INT DEFAULT 0,
last_rescan_result TEXT,
next_scheduled_rescan TIMESTAMPTZ,
last_analyzed_at TIMESTAMPTZ,
evidence_set_hash BYTEA,
graph_slice_hash BYTEA,
provenance_hints JSONB NOT NULL DEFAULT '[]'::jsonb,
best_hypothesis TEXT,
combined_confidence NUMERIC(4,4) CHECK (combined_confidence IS NULL OR (combined_confidence >= 0 AND combined_confidence <= 1)),
primary_suggested_action TEXT
);
""";
await using var command = new NpgsqlCommand(schema, connection);
await command.ExecuteNonQueryAsync();
}
}

View File

@@ -8,6 +8,7 @@
using System.Net;
using System.Net.Http.Json;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Unknowns.Core.Models;
@@ -38,7 +39,10 @@ public sealed class UnknownsEndpointsTests : IClassFixture<WebApplicationFactory
var settings = new Dictionary<string, string?>
{
["ConnectionStrings:UnknownsDb"] =
"Host=localhost;Database=unknowns_test;Username=test;Password=test"
"Host=localhost;Database=unknowns_test;Username=test;Password=test",
["Authority:ResourceServer:Authority"] = "http://localhost",
["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32",
["Authority:ResourceServer:BypassNetworks:1"] = "::1/128"
};
config.AddInMemoryCollection(settings);
});
@@ -56,6 +60,8 @@ public sealed class UnknownsEndpointsTests : IClassFixture<WebApplicationFactory
// Add mock repository
services.AddSingleton(_mockRepository);
});
builder.ConfigureTestServices(UnknownsTestSecurity.Configure);
});
}

View File

@@ -0,0 +1,70 @@
using System.Security.Claims;
using System.Text.Encodings.Web;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Unknowns.WebService.Tests;
internal static class UnknownsTestSecurity
{
public static void Configure(IServiceCollection services)
{
services.AddAuthentication(TestAuthHandler.SchemeName)
.AddScheme<AuthenticationSchemeOptions, TestAuthHandler>(
TestAuthHandler.SchemeName,
_ => { });
services.PostConfigureAll<AuthenticationOptions>(options =>
{
options.DefaultAuthenticateScheme = TestAuthHandler.SchemeName;
options.DefaultChallengeScheme = TestAuthHandler.SchemeName;
options.DefaultScheme = TestAuthHandler.SchemeName;
});
services.RemoveAll<IAuthorizationHandler>();
services.AddSingleton<IAuthorizationHandler, AllowAllAuthorizationHandler>();
}
private sealed class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions>
{
public const string SchemeName = "UnknownsTestScheme";
public TestAuthHandler(
IOptionsMonitor<AuthenticationSchemeOptions> options,
ILoggerFactory logger,
UrlEncoder encoder)
: base(options, logger, encoder)
{
}
protected override Task<AuthenticateResult> HandleAuthenticateAsync()
{
var claims = new[]
{
new Claim("scope", "unknowns.read unknowns.write"),
new Claim("scp", "unknowns.read unknowns.write")
};
var principal = new ClaimsPrincipal(new ClaimsIdentity(claims, SchemeName));
var ticket = new AuthenticationTicket(principal, SchemeName);
return Task.FromResult(AuthenticateResult.Success(ticket));
}
}
private sealed class AllowAllAuthorizationHandler : IAuthorizationHandler
{
public Task HandleAsync(AuthorizationHandlerContext context)
{
foreach (var requirement in context.PendingRequirements.ToList())
{
context.Succeed(requirement);
}
return Task.CompletedTask;
}
}
}