up
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
public sealed record PolicyVexLookupRequest
|
||||
{
|
||||
[JsonPropertyName("advisory_keys")]
|
||||
public IReadOnlyList<string> AdvisoryKeys { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("purls")]
|
||||
public IReadOnlyList<string> Purls { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("providers")]
|
||||
public IReadOnlyList<string> Providers { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("statuses")]
|
||||
public IReadOnlyList<string> Statuses { get; init; } = Array.Empty<string>();
|
||||
|
||||
[Range(1, 500)]
|
||||
[JsonPropertyName("limit")]
|
||||
public int Limit { get; init; } = 200;
|
||||
}
|
||||
|
||||
public sealed record PolicyVexLookupResponse(
|
||||
IReadOnlyList<PolicyVexLookupItem> Results,
|
||||
int TotalStatements,
|
||||
DateTimeOffset GeneratedAtUtc);
|
||||
|
||||
public sealed record PolicyVexLookupItem(
|
||||
string AdvisoryKey,
|
||||
IReadOnlyList<string> Aliases,
|
||||
IReadOnlyList<PolicyVexStatement> Statements);
|
||||
|
||||
public sealed record PolicyVexStatement(
|
||||
string ObservationId,
|
||||
string ProviderId,
|
||||
string Status,
|
||||
string ProductKey,
|
||||
string? Purl,
|
||||
string? Cpe,
|
||||
string? Version,
|
||||
string? Justification,
|
||||
string? Detail,
|
||||
DateTimeOffset FirstSeen,
|
||||
DateTimeOffset LastSeen,
|
||||
VexSignatureMetadata? Signature,
|
||||
IReadOnlyDictionary<string, string> Metadata);
|
||||
@@ -13,6 +13,7 @@ public sealed record VexLinksetListItem(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
|
||||
[property: JsonPropertyName("productKey")] string ProductKey,
|
||||
[property: JsonPropertyName("scope")] VexLinksetScope Scope,
|
||||
[property: JsonPropertyName("providerIds")] IReadOnlyList<string> ProviderIds,
|
||||
[property: JsonPropertyName("statuses")] IReadOnlyList<string> Statuses,
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
|
||||
@@ -38,3 +39,11 @@ public sealed record VexLinksetObservationRef(
|
||||
[property: JsonPropertyName("providerId")] string ProviderId,
|
||||
[property: JsonPropertyName("status")] string Status,
|
||||
[property: JsonPropertyName("confidence")] double? Confidence);
|
||||
|
||||
public sealed record VexLinksetScope(
|
||||
[property: JsonPropertyName("productKey")] string ProductKey,
|
||||
[property: JsonPropertyName("type")] string Type,
|
||||
[property: JsonPropertyName("version")] string? Version,
|
||||
[property: JsonPropertyName("purl")] string? Purl,
|
||||
[property: JsonPropertyName("cpe")] string? Cpe,
|
||||
[property: JsonPropertyName("identifiers")] IReadOnlyList<string> Identifiers);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text.Json.Serialization;
|
||||
@@ -7,11 +8,13 @@ using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core.Canonicalization;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using StellaOps.Excititor.WebService.Telemetry;
|
||||
using DomainVexProductScope = StellaOps.Excititor.Core.Observations.VexProductScope;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
@@ -267,11 +270,13 @@ public static class LinksetEndpoints
|
||||
|
||||
private static VexLinksetListItem ToListItem(VexLinkset linkset)
|
||||
{
|
||||
var scope = BuildScope(linkset.Scope);
|
||||
return new VexLinksetListItem(
|
||||
LinksetId: linkset.LinksetId,
|
||||
Tenant: linkset.Tenant,
|
||||
VulnerabilityId: linkset.VulnerabilityId,
|
||||
ProductKey: linkset.ProductKey,
|
||||
Scope: scope,
|
||||
ProviderIds: linkset.ProviderIds.ToList(),
|
||||
Statuses: linkset.Statuses.ToList(),
|
||||
Aliases: Array.Empty<string>(), // Aliases are in observations, not linksets
|
||||
@@ -289,11 +294,13 @@ public static class LinksetEndpoints
|
||||
|
||||
private static VexLinksetDetailResponse ToDetailResponse(VexLinkset linkset)
|
||||
{
|
||||
var scope = BuildScope(linkset.Scope);
|
||||
return new VexLinksetDetailResponse(
|
||||
LinksetId: linkset.LinksetId,
|
||||
Tenant: linkset.Tenant,
|
||||
VulnerabilityId: linkset.VulnerabilityId,
|
||||
ProductKey: linkset.ProductKey,
|
||||
Scope: scope,
|
||||
ProviderIds: linkset.ProviderIds.ToList(),
|
||||
Statuses: linkset.Statuses.ToList(),
|
||||
Confidence: linkset.Confidence.ToString().ToLowerInvariant(),
|
||||
@@ -343,6 +350,17 @@ public static class LinksetEndpoints
|
||||
var raw = $"{timestamp:O}|{id}";
|
||||
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(raw));
|
||||
}
|
||||
|
||||
private static VexLinksetScope BuildScope(DomainVexProductScope scope)
|
||||
{
|
||||
return new VexLinksetScope(
|
||||
ProductKey: scope.ProductKey,
|
||||
Type: scope.Type,
|
||||
Version: scope.Version,
|
||||
Purl: scope.Purl,
|
||||
Cpe: scope.Cpe,
|
||||
Identifiers: scope.Identifiers.ToList());
|
||||
}
|
||||
}
|
||||
|
||||
// Detail response for single linkset
|
||||
@@ -351,6 +369,7 @@ public sealed record VexLinksetDetailResponse(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
|
||||
[property: JsonPropertyName("productKey")] string ProductKey,
|
||||
[property: JsonPropertyName("scope")] VexLinksetScope Scope,
|
||||
[property: JsonPropertyName("providerIds")] IReadOnlyList<string> ProviderIds,
|
||||
[property: JsonPropertyName("statuses")] IReadOnlyList<string> Statuses,
|
||||
[property: JsonPropertyName("confidence")] string Confidence,
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Canonicalization;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
/// <summary>
|
||||
/// Policy-facing VEX lookup endpoints (EXCITITOR-POLICY-20-001).
|
||||
/// Aggregation-only: returns raw observations/statements without consensus or severity.
|
||||
/// </summary>
|
||||
public static class PolicyEndpoints
|
||||
{
|
||||
public static void MapPolicyEndpoints(this WebApplication app)
|
||||
{
|
||||
app.MapPost("/policy/v1/vex/lookup", LookupVexAsync)
|
||||
.WithName("Policy_VexLookup")
|
||||
.WithDescription("Batch VEX lookup by advisory_key and product (aggregation-only)");
|
||||
}
|
||||
|
||||
private static async Task<IResult> LookupVexAsync(
|
||||
HttpContext context,
|
||||
[FromBody] PolicyVexLookupRequest request,
|
||||
IOptions<VexMongoStorageOptions> storageOptions,
|
||||
[FromServices] IVexClaimStore claimStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// AuthN/Z
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out _, out var tenantError))
|
||||
{
|
||||
return tenantError!;
|
||||
}
|
||||
|
||||
// Validate input
|
||||
if ((request.AdvisoryKeys.Count == 0) && (request.Purls.Count == 0))
|
||||
{
|
||||
return Results.BadRequest(new { error = new { code = "ERR_REQUEST", message = "advisory_keys or purls must be provided" } });
|
||||
}
|
||||
|
||||
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
|
||||
var productCanonicalizer = new VexProductKeyCanonicalizer();
|
||||
|
||||
var canonicalAdvisories = request.AdvisoryKeys
|
||||
.Where(a => !string.IsNullOrWhiteSpace(a))
|
||||
.Select(a => canonicalizer.Canonicalize(a.Trim()))
|
||||
.ToList();
|
||||
|
||||
var canonicalProducts = request.Purls
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p))
|
||||
.Select(p => productCanonicalizer.Canonicalize(p.Trim(), purl: p.Trim()))
|
||||
.ToList();
|
||||
|
||||
// Map requested statuses/providers for filtering
|
||||
var statusFilter = request.Statuses
|
||||
.Select(s => Enum.TryParse<VexClaimStatus>(s, true, out var parsed) ? parsed : (VexClaimStatus?)null)
|
||||
.Where(p => p.HasValue)
|
||||
.Select(p => p!.Value)
|
||||
.ToImmutableHashSet();
|
||||
|
||||
var providerFilter = request.Providers
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p))
|
||||
.Select(p => p.Trim())
|
||||
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var limit = Math.Clamp(request.Limit, 1, 500);
|
||||
var now = timeProvider.GetUtcNow();
|
||||
|
||||
var results = new List<PolicyVexLookupItem>();
|
||||
var totalStatements = 0;
|
||||
|
||||
// For each advisory key, fetch claims and filter by product/provider/status
|
||||
foreach (var advisory in canonicalAdvisories)
|
||||
{
|
||||
var claims = await claimStore
|
||||
.FindByVulnerabilityAsync(advisory.AdvisoryKey, limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var filtered = claims
|
||||
.Where(claim => MatchesProvider(providerFilter, claim))
|
||||
.Where(claim => MatchesStatus(statusFilter, claim))
|
||||
.Where(claim => MatchesProduct(canonicalProducts, claim))
|
||||
.OrderByDescending(claim => claim.LastSeen)
|
||||
.ThenBy(claim => claim.ProviderId, StringComparer.Ordinal)
|
||||
.ThenBy(claim => claim.Product.Key, StringComparer.Ordinal)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
totalStatements += filtered.Count;
|
||||
|
||||
var statements = filtered.Select(MapStatement).ToList();
|
||||
var aliases = advisory.Aliases.ToList();
|
||||
if (!aliases.Contains(advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
aliases.Add(advisory.AdvisoryKey);
|
||||
}
|
||||
|
||||
results.Add(new PolicyVexLookupItem(
|
||||
advisory.AdvisoryKey,
|
||||
aliases,
|
||||
statements));
|
||||
}
|
||||
|
||||
var response = new PolicyVexLookupResponse(results, totalStatements, now);
|
||||
return Results.Ok(response);
|
||||
}
|
||||
|
||||
private static bool MatchesProvider(ISet<string> providers, VexClaim claim)
|
||||
=> providers.Count == 0 || providers.Contains(claim.ProviderId, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static bool MatchesStatus(ISet<VexClaimStatus> statuses, VexClaim claim)
|
||||
=> statuses.Count == 0 || statuses.Contains(claim.Status);
|
||||
|
||||
private static bool MatchesProduct(IEnumerable<VexCanonicalProductKey> requestedProducts, VexClaim claim)
|
||||
{
|
||||
if (!requestedProducts.Any())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return requestedProducts.Any(product =>
|
||||
string.Equals(product.ProductKey, claim.Product.Key, StringComparison.OrdinalIgnoreCase) ||
|
||||
product.Links.Any(link => string.Equals(link.Identifier, claim.Product.Key, StringComparison.OrdinalIgnoreCase)) ||
|
||||
(!string.IsNullOrWhiteSpace(product.Purl) && string.Equals(product.Purl, claim.Product.Purl, StringComparison.OrdinalIgnoreCase)));
|
||||
}
|
||||
|
||||
private static PolicyVexStatement MapStatement(VexClaim claim)
|
||||
{
|
||||
var observationId = $"{claim.ProviderId}:{claim.Document.Digest}";
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["document_digest"] = claim.Document.Digest,
|
||||
["document_uri"] = claim.Document.SourceUri.ToString()
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(claim.Document.Revision))
|
||||
{
|
||||
metadata["document_revision"] = claim.Document.Revision!;
|
||||
}
|
||||
|
||||
return new PolicyVexStatement(
|
||||
ObservationId: observationId,
|
||||
ProviderId: claim.ProviderId,
|
||||
Status: claim.Status.ToString(),
|
||||
ProductKey: claim.Product.Key,
|
||||
Purl: claim.Product.Purl,
|
||||
Cpe: claim.Product.Cpe,
|
||||
Version: claim.Product.Version,
|
||||
Justification: claim.Justification?.ToString(),
|
||||
Detail: claim.Detail,
|
||||
FirstSeen: claim.FirstSeen,
|
||||
LastSeen: claim.LastSeen,
|
||||
Signature: claim.Document.Signature,
|
||||
Metadata: metadata);
|
||||
}
|
||||
|
||||
private static bool TryResolveTenant(
|
||||
HttpContext context,
|
||||
VexMongoStorageOptions options,
|
||||
out string tenant,
|
||||
out IResult? problem)
|
||||
{
|
||||
problem = null;
|
||||
tenant = string.Empty;
|
||||
|
||||
var headerTenant = context.Request.Headers["X-Stella-Tenant"].FirstOrDefault();
|
||||
if (!string.IsNullOrWhiteSpace(headerTenant))
|
||||
{
|
||||
tenant = headerTenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(options.DefaultTenant))
|
||||
{
|
||||
tenant = options.DefaultTenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
else
|
||||
{
|
||||
problem = Results.BadRequest(new
|
||||
{
|
||||
error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header is required" }
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -2282,6 +2282,7 @@ MirrorRegistrationEndpoints.MapMirrorRegistrationEndpoints(app);
|
||||
// Evidence and Attestation APIs (WEB-OBS-53-001, WEB-OBS-54-001)
|
||||
EvidenceEndpoints.MapEvidenceEndpoints(app);
|
||||
AttestationEndpoints.MapAttestationEndpoints(app);
|
||||
PolicyEndpoints.MapPolicyEndpoints(app);
|
||||
|
||||
// Observation and Linkset APIs (EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202)
|
||||
ObservationEndpoints.MapObservationEndpoints(app);
|
||||
|
||||
@@ -7,38 +7,63 @@ namespace StellaOps.Excititor.Worker.Options;
|
||||
/// </summary>
|
||||
public sealed class VexWorkerOrchestratorOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether orchestrator integration is enabled.
|
||||
/// </summary>
|
||||
/// <summary>Whether orchestrator integration is enabled.</summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Interval between heartbeat emissions during job execution.
|
||||
/// </summary>
|
||||
/// <summary>Base address of the Orchestrator WebService (e.g. "https://orch.local/").</summary>
|
||||
public Uri? BaseAddress { get; set; }
|
||||
|
||||
/// <summary>Logical job type registered with Orchestrator.</summary>
|
||||
public string JobType { get; set; } = "exc-vex-ingest";
|
||||
|
||||
/// <summary>Unique worker identifier presented to Orchestrator.</summary>
|
||||
public string WorkerId { get; set; } = "excititor-worker";
|
||||
|
||||
/// <summary>Optional task runner identifier (e.g. host name or pod).</summary>
|
||||
public string? TaskRunnerId { get; set; }
|
||||
|
||||
/// <summary>Tenant header name; defaults to Orchestrator default.</summary>
|
||||
public string TenantHeader { get; set; } = "X-Tenant-Id";
|
||||
|
||||
/// <summary>Tenant value to present when claiming jobs.</summary>
|
||||
public string DefaultTenant { get; set; } = "default";
|
||||
|
||||
/// <summary>API key header name for worker auth.</summary>
|
||||
public string ApiKeyHeader { get; set; } = "X-Worker-Token";
|
||||
|
||||
/// <summary>Optional API key value.</summary>
|
||||
public string? ApiKey { get; set; }
|
||||
|
||||
/// <summary>Optional bearer token value.</summary>
|
||||
public string? BearerToken { get; set; }
|
||||
|
||||
/// <summary>Interval between heartbeat emissions during job execution.</summary>
|
||||
public TimeSpan HeartbeatInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Minimum heartbeat interval (safety floor).
|
||||
/// </summary>
|
||||
/// <summary>Minimum heartbeat interval (safety floor).</summary>
|
||||
public TimeSpan MinHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum heartbeat interval (safety cap).
|
||||
/// </summary>
|
||||
/// <summary>Maximum heartbeat interval (safety cap).</summary>
|
||||
public TimeSpan MaxHeartbeatInterval { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>
|
||||
/// Enable verbose logging for heartbeat/artifact events.
|
||||
/// </summary>
|
||||
/// <summary>Lease duration requested when claiming jobs.</summary>
|
||||
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>Lease extension requested on each heartbeat.</summary>
|
||||
public TimeSpan HeartbeatExtend { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>HTTP request timeout when talking to Orchestrator.</summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(15);
|
||||
|
||||
/// <summary>Enable verbose logging for heartbeat/artifact events.</summary>
|
||||
public bool EnableVerboseLogging { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of artifact hashes to retain in state.
|
||||
/// </summary>
|
||||
/// <summary>Maximum number of artifact hashes to retain in state.</summary>
|
||||
public int MaxArtifactHashes { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Default tenant for worker jobs when not specified.
|
||||
/// </summary>
|
||||
public string DefaultTenant { get; set; } = "default";
|
||||
/// <summary>Emit progress events for artifacts while running.</summary>
|
||||
public bool EmitProgressForArtifacts { get; set; } = true;
|
||||
|
||||
/// <summary>Fallback to local state only when orchestrator is unreachable.</summary>
|
||||
public bool AllowLocalFallback { get; set; } = true;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -13,7 +19,7 @@ namespace StellaOps.Excititor.Worker.Orchestration;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IVexWorkerOrchestratorClient"/>.
|
||||
/// Stores heartbeats and artifacts locally and emits them to the orchestrator registry when configured.
|
||||
/// Stores heartbeats and artifacts locally and, when configured, mirrors them to the Orchestrator worker API.
|
||||
/// </summary>
|
||||
internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
{
|
||||
@@ -21,37 +27,94 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptions<VexWorkerOrchestratorOptions> _options;
|
||||
private readonly ILogger<VexWorkerOrchestratorClient> _logger;
|
||||
private readonly HttpClient? _httpClient;
|
||||
private readonly JsonSerializerOptions _serializerOptions;
|
||||
private VexWorkerCommand? _pendingCommand;
|
||||
private long _commandSequence;
|
||||
|
||||
public VexWorkerOrchestratorClient(
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
TimeProvider timeProvider,
|
||||
IOptions<VexWorkerOrchestratorOptions> options,
|
||||
ILogger<VexWorkerOrchestratorClient> logger)
|
||||
ILogger<VexWorkerOrchestratorClient> logger,
|
||||
HttpClient? httpClient = null)
|
||||
{
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_httpClient = httpClient;
|
||||
_serializerOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
}
|
||||
|
||||
public ValueTask<VexWorkerJobContext> StartJobAsync(
|
||||
public async ValueTask<VexWorkerJobContext> StartJobAsync(
|
||||
string tenant,
|
||||
string connectorId,
|
||||
string? checkpoint,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var runId = Guid.NewGuid();
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
var context = new VexWorkerJobContext(tenant, connectorId, runId, checkpoint, startedAt);
|
||||
var fallbackContext = new VexWorkerJobContext(tenant, connectorId, Guid.NewGuid(), checkpoint, startedAt);
|
||||
|
||||
if (!CanUseRemote())
|
||||
{
|
||||
return fallbackContext;
|
||||
}
|
||||
|
||||
var claimRequest = new ClaimRequest(
|
||||
WorkerId: _options.Value.WorkerId,
|
||||
TaskRunnerId: _options.Value.TaskRunnerId ?? Environment.MachineName,
|
||||
JobType: ResolveJobType(connectorId),
|
||||
LeaseSeconds: ResolveLeaseSeconds(),
|
||||
IdempotencyKey: $"exc-{connectorId}-{startedAt.ToUnixTimeSeconds()}");
|
||||
|
||||
var response = await PostAsync("api/v1/orchestrator/worker/claim", tenant, claimRequest, cancellationToken).ConfigureAwait(false);
|
||||
if (response is null)
|
||||
{
|
||||
return fallbackContext;
|
||||
}
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.NoContent)
|
||||
{
|
||||
_logger.LogInformation("Orchestrator had no jobs for {ConnectorId}; continuing with local execution.", connectorId);
|
||||
return fallbackContext;
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
await HandleErrorResponseAsync("claim", response, connectorId, cancellationToken).ConfigureAwait(false);
|
||||
return fallbackContext;
|
||||
}
|
||||
|
||||
var claim = await DeserializeAsync<ClaimResponse>(response, cancellationToken).ConfigureAwait(false);
|
||||
if (claim is null)
|
||||
{
|
||||
return fallbackContext;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Orchestrator job started: tenant={Tenant} connector={ConnectorId} runId={RunId} checkpoint={Checkpoint}",
|
||||
"Orchestrator job claimed: tenant={Tenant} connector={ConnectorId} jobId={JobId} leaseUntil={LeaseUntil:O}",
|
||||
tenant,
|
||||
connectorId,
|
||||
runId,
|
||||
checkpoint ?? "(none)");
|
||||
claim.JobId,
|
||||
claim.LeaseUntil);
|
||||
|
||||
return ValueTask.FromResult(context);
|
||||
return new VexWorkerJobContext(
|
||||
tenant,
|
||||
connectorId,
|
||||
claim.JobId,
|
||||
checkpoint,
|
||||
startedAt,
|
||||
orchestratorJobId: claim.JobId,
|
||||
orchestratorLeaseId: claim.LeaseId,
|
||||
leaseExpiresAt: claim.LeaseUntil,
|
||||
jobType: claim.JobType,
|
||||
correlationId: claim.CorrelationId,
|
||||
orchestratorRunId: claim.RunId);
|
||||
}
|
||||
|
||||
public async ValueTask SendHeartbeatAsync(
|
||||
@@ -87,6 +150,8 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
heartbeat.Progress,
|
||||
heartbeat.LastArtifactHash);
|
||||
}
|
||||
|
||||
await SendRemoteHeartbeatAsync(context, heartbeat, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask RecordArtifactAsync(
|
||||
@@ -106,7 +171,7 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
: state.DocumentDigests;
|
||||
|
||||
// Add artifact hash if not already tracked (cap to avoid unbounded growth)
|
||||
const int maxDigests = 1000;
|
||||
var maxDigests = Math.Max(1, _options.Value.MaxArtifactHashes);
|
||||
if (!digests.Contains(artifact.Hash))
|
||||
{
|
||||
digests = digests.Length >= maxDigests
|
||||
@@ -129,6 +194,8 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
artifact.Hash,
|
||||
artifact.Kind,
|
||||
artifact.ProviderId);
|
||||
|
||||
await SendRemoteProgressForArtifactAsync(context, artifact, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask CompleteJobAsync(
|
||||
@@ -165,6 +232,8 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
result.DocumentsProcessed,
|
||||
result.ClaimsGenerated,
|
||||
duration);
|
||||
|
||||
await SendRemoteCompletionAsync(context, result, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask FailJobAsync(
|
||||
@@ -202,6 +271,13 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
context.ConnectorId,
|
||||
errorCode,
|
||||
retryAfterSeconds);
|
||||
|
||||
await SendRemoteCompletionAsync(
|
||||
context,
|
||||
new VexWorkerJobResult(0, 0, state.LastCheckpoint, state.LastArtifactHash, now),
|
||||
cancellationToken,
|
||||
success: false,
|
||||
failureReason: Truncate($"{errorCode}: {errorMessage}", 256)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public ValueTask FailJobAsync(
|
||||
@@ -232,16 +308,13 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
// In this local implementation, commands are not externally sourced.
|
||||
// Return Continue to indicate normal processing should continue.
|
||||
// A full orchestrator integration would poll a command queue here.
|
||||
if (!_options.Value.Enabled)
|
||||
{
|
||||
return ValueTask.FromResult<VexWorkerCommand?>(null);
|
||||
}
|
||||
|
||||
// No pending commands in local mode
|
||||
return ValueTask.FromResult<VexWorkerCommand?>(null);
|
||||
var command = Interlocked.Exchange(ref _pendingCommand, null);
|
||||
return ValueTask.FromResult(command);
|
||||
}
|
||||
|
||||
public ValueTask AcknowledgeCommandAsync(
|
||||
@@ -256,7 +329,6 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
context.RunId,
|
||||
commandSequence);
|
||||
|
||||
// In local mode, acknowledgment is a no-op
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
@@ -314,6 +386,12 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
state.ResumeTokens.IsEmpty ? ImmutableDictionary<string, string>.Empty : state.ResumeTokens);
|
||||
}
|
||||
|
||||
private bool CanUseRemote()
|
||||
{
|
||||
var opts = _options.Value;
|
||||
return opts.Enabled && _httpClient is not null && opts.BaseAddress is not null;
|
||||
}
|
||||
|
||||
private static string Truncate(string? value, int maxLength)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value))
|
||||
@@ -325,4 +403,276 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
? value
|
||||
: value[..maxLength];
|
||||
}
|
||||
|
||||
private int ResolveLeaseSeconds()
|
||||
{
|
||||
var seconds = (int)Math.Round(_options.Value.DefaultLeaseDuration.TotalSeconds);
|
||||
return Math.Clamp(seconds, 30, 3600);
|
||||
}
|
||||
|
||||
private int ResolveHeartbeatExtendSeconds()
|
||||
{
|
||||
var opts = _options.Value;
|
||||
var seconds = (int)Math.Round(opts.HeartbeatExtend.TotalSeconds);
|
||||
var min = (int)Math.Round(opts.MinHeartbeatInterval.TotalSeconds);
|
||||
var max = (int)Math.Round(opts.MaxHeartbeatInterval.TotalSeconds);
|
||||
return Math.Clamp(seconds, min, max);
|
||||
}
|
||||
|
||||
private string ResolveJobType(string connectorId)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(_options.Value.JobType)
|
||||
? $"exc-vex-{connectorId}"
|
||||
: _options.Value.JobType;
|
||||
}
|
||||
|
||||
private async ValueTask SendRemoteHeartbeatAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerHeartbeat heartbeat,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!CanUseRemote() || context.OrchestratorJobId is null || context.OrchestratorLeaseId is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var request = new HeartbeatRequest(
|
||||
context.OrchestratorLeaseId.Value,
|
||||
ResolveHeartbeatExtendSeconds(),
|
||||
IdempotencyKey: $"hb-{context.RunId}-{context.Sequence}");
|
||||
|
||||
var response = await PostAsync(
|
||||
$"api/v1/orchestrator/worker/jobs/{context.OrchestratorJobId}/heartbeat",
|
||||
context.Tenant,
|
||||
request,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (response is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var hb = await DeserializeAsync<HeartbeatResponse>(response, cancellationToken).ConfigureAwait(false);
|
||||
if (hb?.LeaseUntil is not null)
|
||||
{
|
||||
context.UpdateLease(hb.LeaseUntil);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await HandleErrorResponseAsync("heartbeat", response, context.ConnectorId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async ValueTask SendRemoteProgressForArtifactAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerArtifact artifact,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!CanUseRemote() || !_options.Value.EmitProgressForArtifacts || context.OrchestratorJobId is null || context.OrchestratorLeaseId is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var metadata = Serialize(new
|
||||
{
|
||||
artifact.Hash,
|
||||
artifact.Kind,
|
||||
artifact.ProviderId,
|
||||
artifact.DocumentId,
|
||||
artifact.CreatedAt
|
||||
});
|
||||
|
||||
var request = new ProgressRequest(
|
||||
context.OrchestratorLeaseId.Value,
|
||||
ProgressPercent: null,
|
||||
Message: $"artifact:{artifact.Kind}",
|
||||
Metadata: metadata,
|
||||
IdempotencyKey: $"artifact-{artifact.Hash}");
|
||||
|
||||
var response = await PostAsync(
|
||||
$"api/v1/orchestrator/worker/jobs/{context.OrchestratorJobId}/progress",
|
||||
context.Tenant,
|
||||
request,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (response is not null && !response.IsSuccessStatusCode)
|
||||
{
|
||||
await HandleErrorResponseAsync("progress", response, context.ConnectorId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask SendRemoteCompletionAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerJobResult result,
|
||||
CancellationToken cancellationToken,
|
||||
bool success = true,
|
||||
string? failureReason = null)
|
||||
{
|
||||
if (!CanUseRemote() || context.OrchestratorJobId is null || context.OrchestratorLeaseId is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var request = new CompleteRequest(
|
||||
context.OrchestratorLeaseId.Value,
|
||||
success,
|
||||
success ? null : failureReason,
|
||||
Artifacts: Array.Empty<ArtifactInput>(),
|
||||
ResultDigest: result.LastArtifactHash,
|
||||
IdempotencyKey: $"complete-{context.RunId}-{context.Sequence}");
|
||||
|
||||
var response = await PostAsync(
|
||||
$"api/v1/orchestrator/worker/jobs/{context.OrchestratorJobId}/complete",
|
||||
context.Tenant,
|
||||
request,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (response is not null && !response.IsSuccessStatusCode)
|
||||
{
|
||||
await HandleErrorResponseAsync("complete", response, context.ConnectorId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage?> PostAsync<TPayload>(
|
||||
string path,
|
||||
string tenant,
|
||||
TPayload payload,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!CanUseRemote())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, path)
|
||||
{
|
||||
Content = new StringContent(JsonSerializer.Serialize(payload, _serializerOptions), Encoding.UTF8, "application/json")
|
||||
};
|
||||
|
||||
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
|
||||
var opts = _options.Value;
|
||||
request.Headers.TryAddWithoutValidation(string.IsNullOrWhiteSpace(opts.TenantHeader) ? "X-Tenant-Id" : opts.TenantHeader, tenant);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(opts.ApiKey))
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(string.IsNullOrWhiteSpace(opts.ApiKeyHeader) ? "X-Worker-Token" : opts.ApiKeyHeader, opts.ApiKey);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(opts.BearerToken))
|
||||
{
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", opts.BearerToken);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return await _httpClient!.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (opts.AllowLocalFallback)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to contact Orchestrator ({Path}); continuing locally.", path);
|
||||
StorePendingCommand(VexWorkerCommandKind.Retry, reason: "orchestrator_unreachable", retryAfterSeconds: 60);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask<T?> DeserializeAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken)
|
||||
{
|
||||
var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
return await JsonSerializer.DeserializeAsync<T>(stream, _serializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task HandleErrorResponseAsync(
|
||||
string stage,
|
||||
HttpResponseMessage response,
|
||||
string connectorId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ErrorResponse? error = null;
|
||||
|
||||
try
|
||||
{
|
||||
error = await DeserializeAsync<ErrorResponse>(response, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore parse issues; fall back to status code handling
|
||||
}
|
||||
|
||||
var retryAfter = error?.RetryAfterSeconds;
|
||||
|
||||
switch (response.StatusCode)
|
||||
{
|
||||
case HttpStatusCode.TooManyRequests:
|
||||
StorePendingCommand(VexWorkerCommandKind.Throttle, reason: error?.Message ?? "rate_limited", retryAfterSeconds: retryAfter ?? 60);
|
||||
break;
|
||||
case HttpStatusCode.Conflict:
|
||||
StorePendingCommand(VexWorkerCommandKind.Retry, reason: error?.Message ?? "lease_conflict", retryAfterSeconds: retryAfter ?? 30);
|
||||
break;
|
||||
case HttpStatusCode.ServiceUnavailable:
|
||||
case HttpStatusCode.BadGateway:
|
||||
case HttpStatusCode.GatewayTimeout:
|
||||
StorePendingCommand(VexWorkerCommandKind.Pause, reason: error?.Message ?? "orchestrator_unavailable", retryAfterSeconds: retryAfter ?? 120);
|
||||
break;
|
||||
default:
|
||||
StorePendingCommand(VexWorkerCommandKind.Retry, reason: error?.Message ?? "orchestrator_error", retryAfterSeconds: retryAfter ?? 30);
|
||||
break;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Orchestrator {Stage} call failed for connector {ConnectorId}: {Status} {Error}",
|
||||
stage,
|
||||
connectorId,
|
||||
response.StatusCode,
|
||||
error?.Message ?? response.ReasonPhrase);
|
||||
}
|
||||
|
||||
private void StorePendingCommand(VexWorkerCommandKind kind, string? reason = null, int? retryAfterSeconds = null)
|
||||
{
|
||||
var issuedAt = _timeProvider.GetUtcNow();
|
||||
var sequence = Interlocked.Increment(ref _commandSequence);
|
||||
var expiresAt = retryAfterSeconds.HasValue ? issuedAt.AddSeconds(retryAfterSeconds.Value) : (DateTimeOffset?)null;
|
||||
|
||||
_pendingCommand = new VexWorkerCommand(
|
||||
kind,
|
||||
sequence,
|
||||
issuedAt,
|
||||
expiresAt,
|
||||
Throttle: kind == VexWorkerCommandKind.Throttle && retryAfterSeconds.HasValue
|
||||
? new VexWorkerThrottleParams(null, null, retryAfterSeconds)
|
||||
: null,
|
||||
Reason: reason);
|
||||
}
|
||||
|
||||
private string Serialize(object value) => JsonSerializer.Serialize(value, _serializerOptions);
|
||||
|
||||
private sealed record ClaimRequest(string WorkerId, string? TaskRunnerId, string? JobType, int? LeaseSeconds, string? IdempotencyKey);
|
||||
|
||||
private sealed record ClaimResponse(
|
||||
Guid JobId,
|
||||
Guid LeaseId,
|
||||
string JobType,
|
||||
string Payload,
|
||||
string PayloadDigest,
|
||||
int Attempt,
|
||||
int MaxAttempts,
|
||||
DateTimeOffset LeaseUntil,
|
||||
string IdempotencyKey,
|
||||
string? CorrelationId,
|
||||
Guid? RunId,
|
||||
string? ProjectId);
|
||||
|
||||
private sealed record HeartbeatRequest(Guid LeaseId, int? ExtendSeconds, string? IdempotencyKey);
|
||||
|
||||
private sealed record HeartbeatResponse(Guid JobId, Guid LeaseId, DateTimeOffset LeaseUntil, bool Acknowledged);
|
||||
|
||||
private sealed record ProgressRequest(Guid LeaseId, double? ProgressPercent, string? Message, string? Metadata, string? IdempotencyKey);
|
||||
|
||||
private sealed record CompleteRequest(Guid LeaseId, bool Success, string? Reason, IReadOnlyList<ArtifactInput>? Artifacts, string? ResultDigest, string? IdempotencyKey);
|
||||
|
||||
private sealed record ArtifactInput(string ArtifactType, string Uri, string Digest, string? MimeType, long? SizeBytes, string? Metadata);
|
||||
|
||||
private sealed record ErrorResponse(string Error, string Message, Guid? JobId, int? RetryAfterSeconds);
|
||||
}
|
||||
|
||||
@@ -109,7 +109,16 @@ services.AddSingleton<PluginCatalog>(provider =>
|
||||
services.AddOptions<VexWorkerOrchestratorOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Worker:Orchestrator"))
|
||||
.ValidateOnStart();
|
||||
services.AddSingleton<IVexWorkerOrchestratorClient, VexWorkerOrchestratorClient>();
|
||||
services.AddHttpClient<IVexWorkerOrchestratorClient, VexWorkerOrchestratorClient>((provider, client) =>
|
||||
{
|
||||
var opts = provider.GetRequiredService<IOptions<VexWorkerOrchestratorOptions>>().Value;
|
||||
if (opts.BaseAddress is not null)
|
||||
{
|
||||
client.BaseAddress = opts.BaseAddress;
|
||||
}
|
||||
|
||||
client.Timeout = opts.RequestTimeout;
|
||||
});
|
||||
services.AddSingleton<VexWorkerHeartbeatService>();
|
||||
|
||||
services.AddSingleton<IVexProviderRunner, DefaultVexProviderRunner>();
|
||||
|
||||
@@ -16,6 +16,7 @@ public sealed record VexLinkset
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
VexProductScope scope,
|
||||
IEnumerable<VexLinksetObservationRefModel> observations,
|
||||
IEnumerable<VexObservationDisagreement>? disagreements = null,
|
||||
DateTimeOffset? createdAt = null,
|
||||
@@ -25,6 +26,7 @@ public sealed record VexLinkset
|
||||
Tenant = VexObservation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
|
||||
VulnerabilityId = VexObservation.EnsureNotNullOrWhiteSpace(vulnerabilityId, nameof(vulnerabilityId));
|
||||
ProductKey = VexObservation.EnsureNotNullOrWhiteSpace(productKey, nameof(productKey));
|
||||
Scope = scope ?? throw new ArgumentNullException(nameof(scope));
|
||||
Observations = NormalizeObservations(observations);
|
||||
Disagreements = NormalizeDisagreements(disagreements);
|
||||
CreatedAt = (createdAt ?? DateTimeOffset.UtcNow).ToUniversalTime();
|
||||
@@ -52,6 +54,11 @@ public sealed record VexLinkset
|
||||
/// </summary>
|
||||
public string ProductKey { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Canonical scope metadata for the product key.
|
||||
/// </summary>
|
||||
public VexProductScope Scope { get; }
|
||||
|
||||
/// <summary>
|
||||
/// References to observations that contribute to this linkset.
|
||||
/// </summary>
|
||||
@@ -154,6 +161,7 @@ public sealed record VexLinkset
|
||||
Tenant,
|
||||
VulnerabilityId,
|
||||
ProductKey,
|
||||
Scope,
|
||||
observations,
|
||||
disagreements,
|
||||
CreatedAt,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using StellaOps.Excititor.Core.Canonicalization;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
@@ -49,6 +50,7 @@ public sealed class VexLinksetExtractionService
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var linksetId = BuildLinksetId(group.Key.VulnerabilityId, group.Key.ProductKey);
|
||||
var scope = BuildScope(group.Key.ProductKey);
|
||||
var obsForGroup = group.Select(x => x.obs);
|
||||
|
||||
var evt = VexLinksetUpdatedEventFactory.Create(
|
||||
@@ -56,6 +58,7 @@ public sealed class VexLinksetExtractionService
|
||||
linksetId,
|
||||
group.Key.VulnerabilityId,
|
||||
group.Key.ProductKey,
|
||||
scope,
|
||||
obsForGroup,
|
||||
disagreements ?? Enumerable.Empty<VexObservationDisagreement>(),
|
||||
now);
|
||||
@@ -69,5 +72,46 @@ public sealed class VexLinksetExtractionService
|
||||
private static string BuildLinksetId(string vulnerabilityId, string productKey)
|
||||
=> $"vex:{vulnerabilityId}:{productKey}".ToLowerInvariant();
|
||||
|
||||
private static VexProductScope BuildScope(string productKey)
|
||||
{
|
||||
var canonicalizer = new VexProductKeyCanonicalizer();
|
||||
try
|
||||
{
|
||||
var canonical = canonicalizer.Canonicalize(productKey);
|
||||
var identifiers = canonical.Links
|
||||
.Where(link => link is not null && !string.IsNullOrWhiteSpace(link.Identifier))
|
||||
.Select(link => link.Identifier.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
var purl = canonical.Links.FirstOrDefault(link => string.Equals(link.Type, "purl", StringComparison.OrdinalIgnoreCase))?.Identifier;
|
||||
var cpe = canonical.Links.FirstOrDefault(link => string.Equals(link.Type, "cpe", StringComparison.OrdinalIgnoreCase))?.Identifier;
|
||||
var version = ExtractVersion(purl ?? canonical.ProductKey);
|
||||
|
||||
return new VexProductScope(
|
||||
ProductKey: canonical.ProductKey,
|
||||
Type: canonical.Scope.ToString().ToLowerInvariant(),
|
||||
Version: version,
|
||||
Purl: purl,
|
||||
Cpe: cpe,
|
||||
Identifiers: identifiers);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return VexProductScope.Unknown(productKey);
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractVersion(string? key)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var at = key.LastIndexOf('@');
|
||||
return at >= 0 && at < key.Length - 1 ? key[(at + 1)..] : null;
|
||||
}
|
||||
|
||||
private static string Normalize(string value) => VexObservation.EnsureNotNullOrWhiteSpace(value, nameof(value));
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ public static class VexLinksetUpdatedEventFactory
|
||||
string linksetId,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
VexProductScope scope,
|
||||
IEnumerable<VexObservation> observations,
|
||||
IEnumerable<VexObservationDisagreement> disagreements,
|
||||
DateTimeOffset createdAtUtc)
|
||||
@@ -62,6 +63,7 @@ public static class VexLinksetUpdatedEventFactory
|
||||
normalizedLinksetId,
|
||||
normalizedVulnerabilityId,
|
||||
normalizedProductKey,
|
||||
scope,
|
||||
observationRefs,
|
||||
disagreementList,
|
||||
createdAtUtc);
|
||||
@@ -151,6 +153,7 @@ public sealed record VexLinksetUpdatedEvent(
|
||||
string LinksetId,
|
||||
string VulnerabilityId,
|
||||
string ProductKey,
|
||||
VexProductScope Scope,
|
||||
ImmutableArray<VexLinksetObservationRefCore> Observations,
|
||||
ImmutableArray<VexObservationDisagreement> Disagreements,
|
||||
DateTimeOffset CreatedAtUtc);
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Canonical scope metadata derived from a product identifier.
|
||||
/// </summary>
|
||||
public sealed record VexProductScope(
|
||||
string ProductKey,
|
||||
string Type,
|
||||
string? Version,
|
||||
string? Purl,
|
||||
string? Cpe,
|
||||
ImmutableArray<string> Identifiers)
|
||||
{
|
||||
public static VexProductScope Unknown(string productKey) =>
|
||||
new(productKey ?? string.Empty, "unknown", null, null, null, ImmutableArray<string>.Empty);
|
||||
}
|
||||
@@ -103,13 +103,25 @@ public sealed record VexWorkerJobContext
|
||||
string connectorId,
|
||||
Guid runId,
|
||||
string? checkpoint,
|
||||
DateTimeOffset startedAt)
|
||||
DateTimeOffset startedAt,
|
||||
Guid? orchestratorJobId = null,
|
||||
Guid? orchestratorLeaseId = null,
|
||||
DateTimeOffset? leaseExpiresAt = null,
|
||||
string? jobType = null,
|
||||
string? correlationId = null,
|
||||
Guid? orchestratorRunId = null)
|
||||
{
|
||||
Tenant = EnsureNotNullOrWhiteSpace(tenant, nameof(tenant));
|
||||
ConnectorId = EnsureNotNullOrWhiteSpace(connectorId, nameof(connectorId));
|
||||
RunId = runId;
|
||||
Checkpoint = checkpoint?.Trim();
|
||||
StartedAt = startedAt;
|
||||
OrchestratorJobId = orchestratorJobId;
|
||||
OrchestratorLeaseId = orchestratorLeaseId;
|
||||
LeaseExpiresAt = leaseExpiresAt;
|
||||
JobType = jobType;
|
||||
CorrelationId = correlationId;
|
||||
OrchestratorRunId = orchestratorRunId;
|
||||
}
|
||||
|
||||
public string Tenant { get; }
|
||||
@@ -117,6 +129,12 @@ public sealed record VexWorkerJobContext
|
||||
public Guid RunId { get; }
|
||||
public string? Checkpoint { get; }
|
||||
public DateTimeOffset StartedAt { get; }
|
||||
public Guid? OrchestratorJobId { get; }
|
||||
public Guid? OrchestratorLeaseId { get; }
|
||||
public DateTimeOffset? LeaseExpiresAt { get; private set; }
|
||||
public string? JobType { get; }
|
||||
public string? CorrelationId { get; }
|
||||
public Guid? OrchestratorRunId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Current sequence number for heartbeats.
|
||||
@@ -128,6 +146,11 @@ public sealed record VexWorkerJobContext
|
||||
/// </summary>
|
||||
public long NextSequence() => ++Sequence;
|
||||
|
||||
/// <summary>
|
||||
/// Updates the tracked lease expiration when the orchestrator extends it.
|
||||
/// </summary>
|
||||
public void UpdateLease(DateTimeOffset leaseUntil) => LeaseExpiresAt = leaseUntil;
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
}
|
||||
|
||||
@@ -57,6 +57,15 @@ internal sealed class MongoVexLinksetEventPublisher : IVexLinksetEventPublisher
|
||||
LinksetId = @event.LinksetId,
|
||||
VulnerabilityId = @event.VulnerabilityId,
|
||||
ProductKey = @event.ProductKey,
|
||||
Scope = new VexLinksetScopeRecord
|
||||
{
|
||||
ProductKey = @event.Scope.ProductKey,
|
||||
Type = @event.Scope.Type,
|
||||
Version = @event.Scope.Version,
|
||||
Purl = @event.Scope.Purl,
|
||||
Cpe = @event.Scope.Cpe,
|
||||
Identifiers = @event.Scope.Identifiers.ToList(),
|
||||
},
|
||||
Observations = @event.Observations
|
||||
.Select(o => new VexLinksetEventObservationRecord
|
||||
{
|
||||
|
||||
@@ -98,6 +98,7 @@ internal sealed class MongoVexLinksetStore : IVexLinksetStore
|
||||
normalizedTenant,
|
||||
normalizedVuln,
|
||||
normalizedProduct,
|
||||
scope: VexProductScope.Unknown(normalizedProduct),
|
||||
observations: Array.Empty<VexLinksetObservationRefModel>(),
|
||||
disagreements: null,
|
||||
createdAt: DateTimeOffset.UtcNow,
|
||||
@@ -275,6 +276,7 @@ internal sealed class MongoVexLinksetStore : IVexLinksetStore
|
||||
LinksetId = linkset.LinksetId,
|
||||
VulnerabilityId = linkset.VulnerabilityId.ToLowerInvariant(),
|
||||
ProductKey = linkset.ProductKey.ToLowerInvariant(),
|
||||
Scope = ToScopeRecord(linkset.Scope),
|
||||
ProviderIds = linkset.ProviderIds.ToList(),
|
||||
Statuses = linkset.Statuses.ToList(),
|
||||
CreatedAt = linkset.CreatedAt.UtcDateTime,
|
||||
@@ -326,14 +328,49 @@ internal sealed class MongoVexLinksetStore : IVexLinksetStore
|
||||
d.Confidence))
|
||||
.ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
|
||||
var scope = record.Scope is not null
|
||||
? ToScope(record.Scope)
|
||||
: VexProductScope.Unknown(record.ProductKey);
|
||||
|
||||
return new VexLinkset(
|
||||
linksetId: record.LinksetId,
|
||||
tenant: record.Tenant,
|
||||
vulnerabilityId: record.VulnerabilityId,
|
||||
productKey: record.ProductKey,
|
||||
scope: scope,
|
||||
observations: observations,
|
||||
disagreements: disagreements,
|
||||
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
|
||||
updatedAt: new DateTimeOffset(record.UpdatedAt, TimeSpan.Zero));
|
||||
}
|
||||
|
||||
private static VexLinksetScopeRecord ToScopeRecord(VexProductScope scope)
|
||||
{
|
||||
return new VexLinksetScopeRecord
|
||||
{
|
||||
ProductKey = scope.ProductKey,
|
||||
Type = scope.Type,
|
||||
Version = scope.Version,
|
||||
Purl = scope.Purl,
|
||||
Cpe = scope.Cpe,
|
||||
Identifiers = scope.Identifiers.ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static VexProductScope ToScope(VexLinksetScopeRecord record)
|
||||
{
|
||||
var identifiers = record.Identifiers?
|
||||
.Where(id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
|
||||
return new VexProductScope(
|
||||
ProductKey: record.ProductKey ?? string.Empty,
|
||||
Type: record.Type ?? "unknown",
|
||||
Version: record.Version,
|
||||
Purl: record.Purl,
|
||||
Cpe: record.Cpe,
|
||||
Identifiers: identifiers);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -484,6 +484,8 @@ internal sealed class VexLinksetRecord
|
||||
|
||||
public string ProductKey { get; set; } = default!;
|
||||
|
||||
public VexLinksetScopeRecord? Scope { get; set; }
|
||||
|
||||
public List<string> ProviderIds { get; set; } = new();
|
||||
|
||||
public List<string> Statuses { get; set; } = new();
|
||||
@@ -1397,6 +1399,8 @@ internal sealed class VexLinksetEventRecord
|
||||
|
||||
public string ProductKey { get; set; } = default!;
|
||||
|
||||
public VexLinksetScopeRecord? Scope { get; set; }
|
||||
|
||||
public List<VexLinksetEventObservationRecord> Observations { get; set; } = new();
|
||||
|
||||
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
|
||||
@@ -1410,6 +1414,24 @@ internal sealed class VexLinksetEventRecord
|
||||
public int ObservationCount { get; set; } = 0;
|
||||
}
|
||||
|
||||
internal sealed class VexLinksetScopeRecord
|
||||
{
|
||||
public string ProductKey { get; set; } = string.Empty;
|
||||
|
||||
public string Type { get; set; } = string.Empty;
|
||||
|
||||
public string? Version { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Purl { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Cpe { get; set; }
|
||||
= null;
|
||||
|
||||
public List<string> Identifiers { get; set; } = new();
|
||||
}
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class VexLinksetEventObservationRecord
|
||||
{
|
||||
|
||||
@@ -32,6 +32,13 @@ public sealed class VexLinksetUpdatedEventFactoryTests
|
||||
linksetId: "link-123",
|
||||
vulnerabilityId: "CVE-2025-0001",
|
||||
productKey: "pkg:demo/app",
|
||||
scope: new VexProductScope(
|
||||
ProductKey: "pkg:demo/app",
|
||||
Type: "package",
|
||||
Version: "1.0.0",
|
||||
Purl: "pkg:demo/app@1.0.0",
|
||||
Cpe: null,
|
||||
Identifiers: ImmutableArray.Create("pkg:demo/app@1.0.0")),
|
||||
observations,
|
||||
disagreements,
|
||||
now);
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class PolicyEndpointsTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task VexLookup_ReturnsStatements_ForAdvisoryAndPurl()
|
||||
{
|
||||
var claims = CreateSampleClaims();
|
||||
|
||||
using var factory = new TestWebApplicationFactory(
|
||||
configureServices: services =>
|
||||
{
|
||||
services.RemoveAll<IVexClaimStore>();
|
||||
services.AddSingleton<IVexClaimStore>(new StubClaimStore(claims));
|
||||
services.AddTestAuthentication();
|
||||
});
|
||||
|
||||
using var client = factory.CreateClient(new() { AllowAutoRedirect = false });
|
||||
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "vex.read");
|
||||
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test");
|
||||
|
||||
var request = new PolicyVexLookupRequest
|
||||
{
|
||||
AdvisoryKeys = new[] { "CVE-2025-1234" },
|
||||
Purls = new[] { "pkg:maven/org.example/app@1.2.3" },
|
||||
Limit = 10
|
||||
};
|
||||
|
||||
var response = await client.PostAsJsonAsync("/policy/v1/vex/lookup", request);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var body = await response.Content.ReadFromJsonAsync<PolicyVexLookupResponse>();
|
||||
|
||||
Assert.NotNull(body);
|
||||
Assert.Single(body!.Results);
|
||||
var result = body.Results.First();
|
||||
Assert.Equal("CVE-2025-1234", result.AdvisoryKey);
|
||||
Assert.Single(result.Statements);
|
||||
var statement = result.Statements.First();
|
||||
Assert.Equal("pkg:maven/org.example/app@1.2.3", statement.Purl);
|
||||
Assert.Equal("affected", statement.Status.ToLowerInvariant());
|
||||
}
|
||||
|
||||
private static IReadOnlyList<VexClaim> CreateSampleClaims()
|
||||
{
|
||||
var now = DateTimeOffset.Parse("2025-12-01T12:00:00Z");
|
||||
var product = new VexProduct(
|
||||
key: "pkg:maven/org.example/app",
|
||||
name: "Example App",
|
||||
version: "1.2.3",
|
||||
purl: "pkg:maven/org.example/app@1.2.3");
|
||||
|
||||
var document = new VexClaimDocument(
|
||||
format: VexDocumentFormat.Csaf,
|
||||
digest: "sha256:deadbeef",
|
||||
sourceUri: new Uri("https://example.org/advisory.json"),
|
||||
revision: "v1",
|
||||
signature: null);
|
||||
|
||||
var claim = new VexClaim(
|
||||
vulnerabilityId: "CVE-2025-1234",
|
||||
providerId: "ghsa",
|
||||
product: product,
|
||||
status: VexClaimStatus.Affected,
|
||||
document: document,
|
||||
firstSeen: now.AddHours(-1),
|
||||
lastSeen: now);
|
||||
|
||||
return new[] { claim };
|
||||
}
|
||||
|
||||
private sealed class StubClaimStore : IVexClaimStore
|
||||
{
|
||||
private readonly IReadOnlyList<VexClaim> _claims;
|
||||
|
||||
public StubClaimStore(IReadOnlyList<VexClaim> claims)
|
||||
{
|
||||
_claims = claims;
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId && c.Product.Key == productKey).ToList());
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId).Take(limit).ToList());
|
||||
}
|
||||
}
|
||||
@@ -41,5 +41,6 @@
|
||||
<Compile Include="GraphTooltipFactoryTests.cs" />
|
||||
<Compile Include="AttestationVerifyEndpointTests.cs" />
|
||||
<Compile Include="OpenApiDiscoveryEndpointTests.cs" />
|
||||
<Compile Include="PolicyEndpointsTests.cs" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -212,6 +212,81 @@ internal static class TestServiceOverrides
|
||||
_records.Add(record);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<AirgapImportRecord?> FindByBundleIdAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
string? mirrorGeneration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var matches = _records
|
||||
.Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(r.BundleId, bundleId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(mirrorGeneration))
|
||||
{
|
||||
matches = matches.Where(r => string.Equals(r.MirrorGeneration, mirrorGeneration, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
var result = matches
|
||||
.OrderByDescending(r => r.MirrorGeneration, StringComparer.OrdinalIgnoreCase)
|
||||
.FirstOrDefault();
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AirgapImportRecord>> ListAsync(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter,
|
||||
int limit,
|
||||
int offset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var query = _records
|
||||
.Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(publisherFilter))
|
||||
{
|
||||
query = query.Where(r => string.Equals(r.Publisher, publisherFilter, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (importedAfter.HasValue)
|
||||
{
|
||||
query = query.Where(r => r.ImportedAt > importedAfter.Value);
|
||||
}
|
||||
|
||||
var result = query
|
||||
.OrderByDescending(r => r.ImportedAt)
|
||||
.Skip(Math.Max(0, offset))
|
||||
.Take(Math.Clamp(limit, 1, 1000))
|
||||
.ToList()
|
||||
.AsReadOnly();
|
||||
|
||||
return Task.FromResult((IReadOnlyList<AirgapImportRecord>)result);
|
||||
}
|
||||
|
||||
public Task<int> CountAsync(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var count = _records
|
||||
.Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(r => string.IsNullOrWhiteSpace(publisherFilter) ||
|
||||
string.Equals(r.Publisher, publisherFilter, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(r => !importedAfter.HasValue || r.ImportedAt > importedAfter.Value)
|
||||
.Count();
|
||||
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubIngestOrchestrator : IVexIngestOrchestrator
|
||||
|
||||
@@ -323,14 +323,17 @@ public sealed class DefaultVexProviderRunnerIntegrationTests : IAsyncLifetime
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopClaimStore : IVexClaimStore
|
||||
{
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
}
|
||||
private sealed class NoopClaimStore : IVexClaimStore
|
||||
{
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
|
||||
@@ -484,14 +484,17 @@ public sealed class DefaultVexProviderRunnerTests
|
||||
=> ValueTask.FromResult<VexRawDocument?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopClaimStore : IVexClaimStore
|
||||
{
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
}
|
||||
private sealed class NoopClaimStore : IVexClaimStore
|
||||
{
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
}
|
||||
|
||||
private sealed class NoopProviderStore : IVexProviderStore
|
||||
{
|
||||
|
||||
@@ -2,6 +2,10 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@@ -18,11 +22,6 @@ public class VexWorkerOrchestratorClientTests
|
||||
{
|
||||
private readonly InMemoryConnectorStateRepository _stateRepository = new();
|
||||
private readonly FakeTimeProvider _timeProvider = new();
|
||||
private readonly IOptions<VexWorkerOrchestratorOptions> _options = Microsoft.Extensions.Options.Options.Create(new VexWorkerOrchestratorOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DefaultTenant = "test-tenant"
|
||||
});
|
||||
|
||||
[Fact]
|
||||
public async Task StartJobAsync_CreatesJobContext()
|
||||
@@ -61,6 +60,172 @@ public class VexWorkerOrchestratorClientTests
|
||||
Assert.NotNull(state.LastHeartbeatAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StartJobAsync_UsesOrchestratorClaim_WhenAvailable()
|
||||
{
|
||||
var jobId = Guid.NewGuid();
|
||||
var leaseId = Guid.NewGuid();
|
||||
|
||||
var handler = new StubHandler(request =>
|
||||
{
|
||||
if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
|
||||
{
|
||||
return JsonResponse(new
|
||||
{
|
||||
jobId,
|
||||
leaseId,
|
||||
jobType = "exc-vex-ingest",
|
||||
payload = "{}",
|
||||
payloadDigest = "sha256:abc",
|
||||
attempt = 1,
|
||||
maxAttempts = 3,
|
||||
leaseUntil = "2025-12-01T12:00:00Z",
|
||||
idempotencyKey = "abc",
|
||||
correlationId = "corr-1",
|
||||
runId = (Guid?)null,
|
||||
projectId = (string?)null
|
||||
});
|
||||
}
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.NotFound);
|
||||
});
|
||||
|
||||
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
|
||||
var client = CreateClient(httpClient, opts => opts.BaseAddress = httpClient.BaseAddress);
|
||||
|
||||
var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
|
||||
|
||||
Assert.Equal(jobId, context.RunId);
|
||||
Assert.Equal(jobId, context.OrchestratorJobId);
|
||||
Assert.Equal(leaseId, context.OrchestratorLeaseId);
|
||||
Assert.Contains(handler.Requests, r => r.RequestUri!.AbsolutePath.EndsWith("/claim"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendHeartbeatAsync_ExtendsLeaseViaOrchestrator()
|
||||
{
|
||||
var jobId = Guid.NewGuid();
|
||||
var leaseId = Guid.NewGuid();
|
||||
var leaseUntil = DateTimeOffset.Parse("2025-12-01T12:05:00Z");
|
||||
|
||||
var handler = new StubHandler(request =>
|
||||
{
|
||||
if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
|
||||
{
|
||||
return JsonResponse(new
|
||||
{
|
||||
jobId,
|
||||
leaseId,
|
||||
jobType = "exc-vex-ingest",
|
||||
payload = "{}",
|
||||
payloadDigest = "sha256:abc",
|
||||
attempt = 1,
|
||||
maxAttempts = 3,
|
||||
leaseUntil = "2025-12-01T12:00:00Z",
|
||||
idempotencyKey = "abc",
|
||||
correlationId = "corr-1",
|
||||
runId = (Guid?)null,
|
||||
projectId = (string?)null
|
||||
});
|
||||
}
|
||||
|
||||
if (request.RequestUri!.AbsolutePath.Contains("/heartbeat"))
|
||||
{
|
||||
return JsonResponse(new
|
||||
{
|
||||
jobId,
|
||||
leaseId,
|
||||
leaseUntil,
|
||||
acknowledged = true
|
||||
});
|
||||
}
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.NotFound);
|
||||
});
|
||||
|
||||
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
|
||||
var client = CreateClient(httpClient, opts =>
|
||||
{
|
||||
opts.BaseAddress = httpClient.BaseAddress;
|
||||
opts.HeartbeatExtend = TimeSpan.FromSeconds(45);
|
||||
});
|
||||
|
||||
var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
|
||||
|
||||
var heartbeat = new VexWorkerHeartbeat(
|
||||
VexWorkerHeartbeatStatus.Running,
|
||||
Progress: 10,
|
||||
QueueDepth: null,
|
||||
LastArtifactHash: "sha256:abc",
|
||||
LastArtifactKind: "vex-document",
|
||||
ErrorCode: null,
|
||||
RetryAfterSeconds: null);
|
||||
|
||||
await client.SendHeartbeatAsync(context, heartbeat);
|
||||
|
||||
Assert.Equal(leaseUntil, context.LeaseExpiresAt);
|
||||
Assert.Contains(handler.Requests, r => r.RequestUri!.AbsolutePath.Contains("/heartbeat"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendHeartbeatAsync_StoresThrottleCommand_On429()
|
||||
{
|
||||
var jobId = Guid.NewGuid();
|
||||
var leaseId = Guid.NewGuid();
|
||||
|
||||
var handler = new StubHandler(request =>
|
||||
{
|
||||
if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
|
||||
{
|
||||
return JsonResponse(new
|
||||
{
|
||||
jobId,
|
||||
leaseId,
|
||||
jobType = "exc-vex-ingest",
|
||||
payload = "{}",
|
||||
payloadDigest = "sha256:abc",
|
||||
attempt = 1,
|
||||
maxAttempts = 3,
|
||||
leaseUntil = "2025-12-01T12:00:00Z",
|
||||
idempotencyKey = "abc",
|
||||
correlationId = "corr-1",
|
||||
runId = (Guid?)null,
|
||||
projectId = (string?)null
|
||||
});
|
||||
}
|
||||
|
||||
if (request.RequestUri!.AbsolutePath.Contains("/heartbeat"))
|
||||
{
|
||||
var error = new { error = "rate_limited", message = "slow down", jobId, retryAfterSeconds = 15 };
|
||||
return JsonResponse(error, HttpStatusCode.TooManyRequests);
|
||||
}
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.NotFound);
|
||||
});
|
||||
|
||||
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
|
||||
var client = CreateClient(httpClient, opts => opts.BaseAddress = httpClient.BaseAddress);
|
||||
|
||||
var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
|
||||
|
||||
var heartbeat = new VexWorkerHeartbeat(
|
||||
VexWorkerHeartbeatStatus.Running,
|
||||
Progress: 5,
|
||||
QueueDepth: null,
|
||||
LastArtifactHash: null,
|
||||
LastArtifactKind: null,
|
||||
ErrorCode: null,
|
||||
RetryAfterSeconds: null);
|
||||
|
||||
await client.SendHeartbeatAsync(context, heartbeat);
|
||||
|
||||
var command = await client.GetPendingCommandAsync(context);
|
||||
|
||||
Assert.NotNull(command);
|
||||
Assert.Equal(VexWorkerCommandKind.Throttle, command!.Kind);
|
||||
Assert.Equal(15, command.Throttle?.CooldownSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordArtifactAsync_TracksArtifactHash()
|
||||
{
|
||||
@@ -140,12 +305,25 @@ public class VexWorkerOrchestratorClientTests
|
||||
Assert.Equal(3, context.NextSequence());
|
||||
}
|
||||
|
||||
private VexWorkerOrchestratorClient CreateClient()
|
||||
=> new(
|
||||
private VexWorkerOrchestratorClient CreateClient(
|
||||
HttpClient? httpClient = null,
|
||||
Action<VexWorkerOrchestratorOptions>? configure = null)
|
||||
{
|
||||
var opts = new VexWorkerOrchestratorOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DefaultTenant = "test-tenant"
|
||||
};
|
||||
|
||||
configure?.Invoke(opts);
|
||||
|
||||
return new VexWorkerOrchestratorClient(
|
||||
_stateRepository,
|
||||
_timeProvider,
|
||||
_options,
|
||||
NullLogger<VexWorkerOrchestratorClient>.Instance);
|
||||
Microsoft.Extensions.Options.Options.Create(opts),
|
||||
NullLogger<VexWorkerOrchestratorClient>.Instance,
|
||||
httpClient);
|
||||
}
|
||||
|
||||
private sealed class FakeTimeProvider : TimeProvider
|
||||
{
|
||||
@@ -175,4 +353,31 @@ public class VexWorkerOrchestratorClientTests
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexConnectorState>>(_states.Values.ToList());
|
||||
}
|
||||
|
||||
private sealed class StubHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Func<HttpRequestMessage, HttpResponseMessage> _responder;
|
||||
|
||||
public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder)
|
||||
{
|
||||
_responder = responder;
|
||||
}
|
||||
|
||||
public List<HttpRequestMessage> Requests { get; } = new();
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(request);
|
||||
return Task.FromResult(_responder(request));
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpResponseMessage JsonResponse(object payload, HttpStatusCode status = HttpStatusCode.OK)
|
||||
{
|
||||
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
|
||||
return new HttpResponseMessage(status)
|
||||
{
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user