feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies. - Documented roles and guidelines in AGENTS.md for Scheduler module. - Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs. - Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics. - Developed API endpoints for managing resolver jobs and retrieving metrics. - Defined models for resolver job requests and responses. - Integrated dependency injection for resolver job services. - Implemented ImpactIndexSnapshot for persisting impact index data. - Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring. - Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService. - Created dotnet-filter.sh script to handle command-line arguments for dotnet. - Established nuget-prime project for managing package downloads.
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
public interface IResolverJobService
|
||||
{
|
||||
Task<ResolverJobResponse> CreateAsync(string tenantId, ResolverJobRequest request, CancellationToken cancellationToken);
|
||||
Task<ResolverJobResponse?> GetAsync(string tenantId, string jobId, CancellationToken cancellationToken);
|
||||
ResolverBacklogMetricsResponse ComputeMetrics(string tenantId);
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight in-memory resolver job service to satisfy API contract and rate-limit callers.
|
||||
/// Suitable for stub/air-gap scenarios; replace with Mongo-backed implementation when ready.
|
||||
/// </summary>
|
||||
public sealed class InMemoryResolverJobService : IResolverJobService
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ResolverJobResponse> _store = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<string, List<DateTimeOffset>> _tenantCreates = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private const int MaxJobsPerMinute = 60;
|
||||
|
||||
public InMemoryResolverJobService(TimeProvider? timeProvider = null)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task<ResolverJobResponse> CreateAsync(string tenantId, ResolverJobRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ValidateRequest(request);
|
||||
|
||||
EnforceRateLimit(tenantId);
|
||||
|
||||
var id = GenerateId(tenantId, request.ArtifactId, request.PolicyId);
|
||||
var created = _timeProvider.GetUtcNow();
|
||||
|
||||
var response = new ResolverJobResponse(
|
||||
id,
|
||||
request.ArtifactId.Trim(),
|
||||
request.PolicyId.Trim(),
|
||||
"queued",
|
||||
created,
|
||||
CompletedAt: null,
|
||||
request.CorrelationId,
|
||||
request.Metadata ?? new Dictionary<string, string>());
|
||||
|
||||
_store[id] = response;
|
||||
TrackCreate(tenantId, created);
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
|
||||
public Task<ResolverJobResponse?> GetAsync(string tenantId, string jobId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
|
||||
|
||||
_store.TryGetValue(jobId, out var response);
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
|
||||
public ResolverBacklogMetricsResponse ComputeMetrics(string tenantId)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var pending = new List<ResolverJobResponse>();
|
||||
var completed = new List<ResolverJobResponse>();
|
||||
|
||||
foreach (var job in _store.Values)
|
||||
{
|
||||
if (string.Equals(job.Status, "completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
completed.Add(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
pending.Add(job);
|
||||
}
|
||||
}
|
||||
|
||||
var lagEntries = completed
|
||||
.Where(j => j.CompletedAt is not null)
|
||||
.Select(j => new ResolverLagEntry(
|
||||
j.Id,
|
||||
j.CompletedAt!.Value,
|
||||
Math.Max((j.CompletedAt!.Value - j.CreatedAt).TotalSeconds, 0d),
|
||||
j.CorrelationId,
|
||||
j.ArtifactId,
|
||||
j.PolicyId))
|
||||
.OrderByDescending(e => e.CompletedAt)
|
||||
.ToList();
|
||||
|
||||
return new ResolverBacklogMetricsResponse(
|
||||
tenantId,
|
||||
Pending: pending.Count,
|
||||
Running: 0,
|
||||
Completed: completed.Count,
|
||||
Failed: 0,
|
||||
MinLagSeconds: lagEntries.Count == 0 ? null : lagEntries.Min(e => (double?)e.LagSeconds),
|
||||
MaxLagSeconds: lagEntries.Count == 0 ? null : lagEntries.Max(e => (double?)e.LagSeconds),
|
||||
AverageLagSeconds: lagEntries.Count == 0 ? null : lagEntries.Average(e => e.LagSeconds),
|
||||
RecentCompleted: lagEntries.Take(5).ToList());
|
||||
}
|
||||
|
||||
private static void ValidateRequest(ResolverJobRequest request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.ArtifactId))
|
||||
{
|
||||
throw new ValidationException("artifactId is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.PolicyId))
|
||||
{
|
||||
throw new ValidationException("policyId is required.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string GenerateId(string tenantId, string artifactId, string policyId)
|
||||
{
|
||||
var raw = $"{tenantId}:{artifactId}:{policyId}:{Guid.NewGuid():N}";
|
||||
return "resolver-" + Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(raw))).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private void EnforceRateLimit(string tenantId)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var cutoff = now.AddMinutes(-1);
|
||||
var list = _tenantCreates.GetOrAdd(tenantId, static _ => new List<DateTimeOffset>());
|
||||
lock (list)
|
||||
{
|
||||
list.RemoveAll(ts => ts < cutoff);
|
||||
if (list.Count >= MaxJobsPerMinute)
|
||||
{
|
||||
throw new InvalidOperationException("resolver job rate limit exceeded");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void TrackCreate(string tenantId, DateTimeOffset timestamp)
|
||||
{
|
||||
var list = _tenantCreates.GetOrAdd(tenantId, static _ => new List<DateTimeOffset>());
|
||||
lock (list)
|
||||
{
|
||||
list.Add(timestamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
public interface IResolverBacklogNotifier
|
||||
{
|
||||
void NotifyIfBreached(ResolverBacklogMetricsResponse metrics);
|
||||
}
|
||||
|
||||
internal sealed class LoggingResolverBacklogNotifier : IResolverBacklogNotifier
|
||||
{
|
||||
private readonly ILogger<LoggingResolverBacklogNotifier> _logger;
|
||||
private readonly int _threshold;
|
||||
|
||||
public LoggingResolverBacklogNotifier(ILogger<LoggingResolverBacklogNotifier> logger, int threshold = 100)
|
||||
{
|
||||
_logger = logger;
|
||||
_threshold = threshold;
|
||||
}
|
||||
|
||||
public void NotifyIfBreached(ResolverBacklogMetricsResponse metrics)
|
||||
{
|
||||
if (metrics.Pending > _threshold)
|
||||
{
|
||||
_logger.LogWarning("resolver backlog threshold exceeded: {Pending} pending (threshold {Threshold})", metrics.Pending, _threshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Scheduler.Queue;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
internal interface IResolverBacklogService
|
||||
{
|
||||
ResolverBacklogSummary GetSummary();
|
||||
}
|
||||
|
||||
internal sealed class ResolverBacklogService : IResolverBacklogService
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ResolverBacklogService(TimeProvider? timeProvider = null)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public ResolverBacklogSummary GetSummary()
|
||||
{
|
||||
var samples = SchedulerQueueMetrics.CaptureDepthSamples();
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return new ResolverBacklogSummary(_timeProvider.GetUtcNow(), 0, 0, ImmutableArray<ResolverBacklogEntry>.Empty);
|
||||
}
|
||||
|
||||
long total = 0;
|
||||
long max = 0;
|
||||
var builder = ImmutableArray.CreateBuilder<ResolverBacklogEntry>(samples.Count);
|
||||
foreach (var sample in samples)
|
||||
{
|
||||
total += sample.Depth;
|
||||
if (sample.Depth > max)
|
||||
{
|
||||
max = sample.Depth;
|
||||
}
|
||||
builder.Add(new ResolverBacklogEntry(sample.Transport, sample.Queue, sample.Depth));
|
||||
}
|
||||
|
||||
return new ResolverBacklogSummary(_timeProvider.GetUtcNow(), total, max, builder.ToImmutable());
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record ResolverBacklogSummary(
|
||||
DateTimeOffset ObservedAt,
|
||||
long TotalDepth,
|
||||
long MaxDepth,
|
||||
IReadOnlyList<ResolverBacklogEntry> Queues);
|
||||
|
||||
public sealed record ResolverBacklogEntry(string Transport, string Queue, long Depth);
|
||||
@@ -0,0 +1,101 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Scheduler.WebService.Auth;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
public static class ResolverJobEndpointExtensions
|
||||
{
|
||||
private const string ScopeWrite = StellaOpsScopes.EffectiveWrite;
|
||||
private const string ScopeRead = StellaOpsScopes.FindingsRead;
|
||||
|
||||
public static void MapResolverJobEndpoints(this IEndpointRouteBuilder builder)
|
||||
{
|
||||
var group = builder.MapGroup("/api/v1/scheduler/vuln/resolver");
|
||||
group.MapPost("/jobs", CreateJobAsync);
|
||||
group.MapGet("/jobs/{jobId}", GetJobAsync);
|
||||
group.MapGet("/metrics", GetLagMetricsAsync);
|
||||
}
|
||||
|
||||
internal static async Task<IResult> CreateJobAsync(
|
||||
[FromBody] ResolverJobRequest request,
|
||||
HttpContext httpContext,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer authorizer,
|
||||
[FromServices] IResolverJobService jobService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
authorizer.EnsureScope(httpContext, ScopeWrite);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var job = await jobService.CreateAsync(tenant.TenantId, request, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Created($"/api/v1/scheduler/vuln/resolver/jobs/{job.Id}", job);
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
catch (ValidationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status400BadRequest);
|
||||
}
|
||||
}
|
||||
|
||||
internal static async Task<IResult> GetJobAsync(
|
||||
string jobId,
|
||||
HttpContext httpContext,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer authorizer,
|
||||
[FromServices] IResolverJobService jobService,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
authorizer.EnsureScope(httpContext, ScopeRead);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var job = await jobService.GetAsync(tenant.TenantId, jobId, cancellationToken).ConfigureAwait(false);
|
||||
return job is null ? Results.NotFound() : Results.Ok(job);
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
}
|
||||
|
||||
internal static IResult GetLagMetricsAsync(
|
||||
HttpContext httpContext,
|
||||
[FromServices] ITenantContextAccessor tenantAccessor,
|
||||
[FromServices] IScopeAuthorizer authorizer,
|
||||
[FromServices] IResolverJobService jobService,
|
||||
[FromServices] IResolverBacklogService backlogService,
|
||||
[FromServices] IResolverBacklogNotifier backlogNotifier)
|
||||
{
|
||||
try
|
||||
{
|
||||
authorizer.EnsureScope(httpContext, ScopeRead);
|
||||
var tenant = tenantAccessor.GetTenant(httpContext);
|
||||
var metrics = jobService.ComputeMetrics(tenant.TenantId);
|
||||
var backlog = backlogService.GetSummary();
|
||||
backlogNotifier.NotifyIfBreached(metrics with { Pending = (int)backlog.TotalDepth });
|
||||
return Results.Ok(new { jobs = metrics, backlog });
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
public sealed record ResolverJobRequest(
|
||||
[property: Required]
|
||||
string ArtifactId,
|
||||
[property: Required]
|
||||
string PolicyId,
|
||||
string? CorrelationId = null,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null)
|
||||
{
|
||||
public ResolverJobRequest() : this(string.Empty, string.Empty, null, null) { }
|
||||
}
|
||||
|
||||
public sealed record ResolverJobResponse(
|
||||
string Id,
|
||||
string ArtifactId,
|
||||
string PolicyId,
|
||||
string Status,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset? CompletedAt,
|
||||
string? CorrelationId,
|
||||
IReadOnlyDictionary<string, string>? Metadata);
|
||||
|
||||
public sealed record ResolverBacklogMetricsResponse(
|
||||
string TenantId,
|
||||
int Pending,
|
||||
int Running,
|
||||
int Completed,
|
||||
int Failed,
|
||||
double? MinLagSeconds,
|
||||
double? MaxLagSeconds,
|
||||
double? AverageLagSeconds,
|
||||
IReadOnlyList<ResolverLagEntry> RecentCompleted);
|
||||
|
||||
public sealed record ResolverLagEntry(
|
||||
string JobId,
|
||||
DateTimeOffset CompletedAt,
|
||||
double LagSeconds,
|
||||
string? CorrelationId,
|
||||
string? ArtifactId,
|
||||
string? PolicyId);
|
||||
@@ -0,0 +1,16 @@
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace StellaOps.Scheduler.WebService.VulnerabilityResolverJobs;
|
||||
|
||||
public static class ResolverJobServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddResolverJobServices(this IServiceCollection services)
|
||||
{
|
||||
services.AddSingleton<IResolverJobService, InMemoryResolverJobService>();
|
||||
services.AddSingleton<IResolverBacklogService, ResolverBacklogService>();
|
||||
services.AddSingleton<IResolverBacklogNotifier, LoggingResolverBacklogNotifier>();
|
||||
return services;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user