feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies.
- Documented roles and guidelines in AGENTS.md for Scheduler module.
- Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs.
- Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics.
- Developed API endpoints for managing resolver jobs and retrieving metrics.
- Defined models for resolver job requests and responses.
- Integrated dependency injection for resolver job services.
- Implemented ImpactIndexSnapshot for persisting impact index data.
- Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring.
- Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService.
- Created dotnet-filter.sh script to handle command-line arguments for dotnet.
- Established nuget-prime project for managing package downloads.
This commit is contained in:
master
2025-11-18 07:52:15 +02:00
parent e69b57d467
commit 8355e2ff75
299 changed files with 13293 additions and 2444 deletions

View File

@@ -107,21 +107,78 @@ public sealed class FixtureImpactIndex : IImpactIndex
return CreateImpactSet(state, selector, Enumerable.Empty<FixtureMatch>(), usageOnly);
}
public async ValueTask<ImpactSet> ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default)
{
public async ValueTask<ImpactSet> ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(selector);
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
var matches = state.ImagesByDigest.Values
.Select(image => new FixtureMatch(image, image.UsedByEntrypoint))
.Where(match => !usageOnly || match.UsedByEntrypoint);
return CreateImpactSet(state, selector, matches, usageOnly);
}
var matches = state.ImagesByDigest.Values
.Select(image => new FixtureMatch(image, image.UsedByEntrypoint))
.Where(match => !usageOnly || match.UsedByEntrypoint);
return CreateImpactSet(state, selector, matches, usageOnly);
}
public ValueTask RemoveAsync(string imageDigest, CancellationToken cancellationToken = default)
{
// Fixture-backed index is immutable; removals are ignored.
return ValueTask.CompletedTask;
}
public async ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
{
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
var images = state.ImagesByDigest.Values
.OrderBy(image => image.Digest, StringComparer.OrdinalIgnoreCase)
.Select((image, index) => new ImpactImageRecord(
index,
"fixture",
image.Digest,
image.Registry,
image.Repository,
image.Namespaces,
image.Tags,
image.Labels,
image.GeneratedAt,
image.Components.Select(c => c.Purl).ToImmutableArray(),
image.Components.Where(c => c.UsedByEntrypoint).Select(c => c.Purl).ToImmutableArray()))
.ToImmutableArray();
var contains = images
.SelectMany(img => img.Components.Select(purl => (purl, img.ImageId)))
.GroupBy(pair => pair.purl, StringComparer.OrdinalIgnoreCase)
.ToImmutableDictionary(
g => g.Key,
g => g.Select(p => p.ImageId).Distinct().OrderBy(id => id).ToImmutableArray(),
StringComparer.OrdinalIgnoreCase);
var usedBy = images
.SelectMany(img => img.EntrypointComponents.Select(purl => (purl, img.ImageId)))
.GroupBy(pair => pair.purl, StringComparer.OrdinalIgnoreCase)
.ToImmutableDictionary(
g => g.Key,
g => g.Select(p => p.ImageId).Distinct().OrderBy(id => id).ToImmutableArray(),
StringComparer.OrdinalIgnoreCase);
return new ImpactIndexSnapshot(
state.GeneratedAt,
state.SnapshotId,
images,
contains,
usedBy);
}
public ValueTask RestoreSnapshotAsync(ImpactIndexSnapshot snapshot, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(snapshot);
// Fixture index remains immutable; restoration is a no-op.
return ValueTask.CompletedTask;
}
private async Task<FixtureIndexState> EnsureInitializedAsync(CancellationToken cancellationToken)
{

View File

@@ -39,8 +39,29 @@ public interface IImpactIndex
/// <param name="selector">Selector scoping the query.</param>
/// <param name="usageOnly">When true, restricts results to images with entrypoint usage.</param>
/// <param name="cancellationToken">Cancellation token.</param>
ValueTask<ImpactSet> ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default);
}
ValueTask<ImpactSet> ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default);
/// <summary>
/// Removes an image digest and its component mappings from the index.
/// Used when an image is deleted or aged out.
/// </summary>
ValueTask RemoveAsync(
string imageDigest,
CancellationToken cancellationToken = default);
/// <summary>
/// Creates a compacted snapshot of the index for persistence (e.g., RocksDB/Redis).
/// </summary>
ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(
CancellationToken cancellationToken = default);
/// <summary>
/// Restores index state from a previously persisted snapshot.
/// </summary>
ValueTask RestoreSnapshotAsync(
ImpactIndexSnapshot snapshot,
CancellationToken cancellationToken = default);
}

View File

@@ -3,7 +3,7 @@ using System.Collections.Immutable;
namespace StellaOps.Scheduler.ImpactIndex;
internal sealed record ImpactImageRecord(
public sealed record ImpactImageRecord(
int ImageId,
string TenantId,
string Digest,

View File

@@ -0,0 +1,37 @@
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scheduler.ImpactIndex;
/// <summary>
/// Serializable snapshot for persisting the ImpactIndex (e.g., RocksDB/Redis).
/// Contains compacted image IDs and per-purl bitmap membership.
/// </summary>
public sealed record ImpactIndexSnapshot(
DateTimeOffset GeneratedAt,
string SnapshotId,
ImmutableArray<ImpactImageRecord> Images,
ImmutableDictionary<string, ImmutableArray<int>> ContainsByPurl,
ImmutableDictionary<string, ImmutableArray<int>> UsedByEntrypointByPurl)
{
public static byte[] ToBytes(ImpactIndexSnapshot snapshot)
{
var options = SerializerOptions;
return JsonSerializer.SerializeToUtf8Bytes(snapshot, options);
}
public static ImpactIndexSnapshot FromBytes(ReadOnlySpan<byte> payload)
{
var options = SerializerOptions;
var snapshot = JsonSerializer.Deserialize<ImpactIndexSnapshot>(payload, options);
return snapshot ?? throw new InvalidOperationException("ImpactIndexSnapshot payload could not be deserialized.");
}
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
}

View File

@@ -23,16 +23,41 @@ public sealed class RoaringImpactIndex : IImpactIndex
private readonly Dictionary<string, int> _imageIds = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<int, ImpactImageRecord> _images = new();
private readonly Dictionary<string, RoaringBitmap> _containsByPurl = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger<RoaringImpactIndex> _logger;
private readonly TimeProvider _timeProvider;
private string? _snapshotId;
private readonly ILogger<RoaringImpactIndex> _logger;
private readonly TimeProvider _timeProvider;
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public ValueTask RemoveAsync(string imageDigest, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
lock (_gate)
{
if (!_imageIds.TryGetValue(imageDigest, out var imageId))
{
return ValueTask.CompletedTask;
}
if (_images.TryGetValue(imageId, out var record))
{
RemoveImageComponents(record);
_images.Remove(imageId);
}
_imageIds.Remove(imageDigest);
_snapshotId = null;
}
return ValueTask.CompletedTask;
}
public async Task IngestAsync(ImpactIndexIngestionRequest request, CancellationToken cancellationToken = default)
{
@@ -130,11 +155,108 @@ public sealed class RoaringImpactIndex : IImpactIndex
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(CreateEmptyImpactSet(selector, usageOnly));
public ValueTask<ImpactSet> ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
public ValueTask<ImpactSet> ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
public ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
lock (_gate)
{
var orderedImages = _images
.Values
.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase)
.ThenBy(img => img.Repository, StringComparer.OrdinalIgnoreCase)
.ToArray();
var idMap = orderedImages
.Select((image, index) => (image.ImageId, NewId: index))
.ToDictionary(tuple => tuple.ImageId, tuple => tuple.NewId);
var compactedImages = orderedImages
.Select(image => image with { ImageId = idMap[image.ImageId] })
.ToImmutableArray();
ImmutableDictionary<string, ImmutableArray<int>> CompactBitmaps(Dictionary<string, RoaringBitmap> source)
{
var builder = ImmutableDictionary.CreateBuilder<string, ImmutableArray<int>>(StringComparer.OrdinalIgnoreCase);
foreach (var (key, bitmap) in source)
{
var remapped = bitmap
.Select(id => idMap.TryGetValue(id, out var newId) ? newId : (int?)null)
.Where(id => id.HasValue)
.Select(id => id!.Value)
.Distinct()
.OrderBy(id => id)
.ToImmutableArray();
if (remapped.Length > 0)
{
builder[key] = remapped;
}
}
return builder.ToImmutable();
}
var contains = CompactBitmaps(_containsByPurl);
var usedBy = CompactBitmaps(_usedByEntrypointByPurl);
var generatedAt = orderedImages.Length == 0
? _timeProvider.GetUtcNow()
: orderedImages.Max(img => img.GeneratedAt);
var snapshotId = ComputeSnapshotId(compactedImages, contains, usedBy);
_snapshotId = snapshotId;
var snapshot = new ImpactIndexSnapshot(
generatedAt,
snapshotId,
compactedImages,
contains,
usedBy);
return ValueTask.FromResult(snapshot);
}
}
public ValueTask RestoreSnapshotAsync(ImpactIndexSnapshot snapshot, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(snapshot);
cancellationToken.ThrowIfCancellationRequested();
lock (_gate)
{
_images.Clear();
_imageIds.Clear();
_containsByPurl.Clear();
_usedByEntrypointByPurl.Clear();
foreach (var image in snapshot.Images)
{
_images[image.ImageId] = image;
_imageIds[image.Digest] = image.ImageId;
}
foreach (var kvp in snapshot.ContainsByPurl)
{
_containsByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
}
foreach (var kvp in snapshot.UsedByEntrypointByPurl)
{
_usedByEntrypointByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
}
_snapshotId = snapshot.SnapshotId;
}
return ValueTask.CompletedTask;
}
private ImpactSet ResolveByPurlsCore(IEnumerable<string> purls, bool usageOnly, Selector selector)
{
@@ -231,27 +353,27 @@ public sealed class RoaringImpactIndex : IImpactIndex
var generatedAt = latestGeneratedAt == DateTimeOffset.MinValue ? _timeProvider.GetUtcNow() : latestGeneratedAt;
return new ImpactSet(
selector,
images.ToImmutableArray(),
usageOnly,
generatedAt,
images.Count,
snapshotId: null,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
return new ImpactSet(
selector,
images.ToImmutableArray(),
usageOnly,
generatedAt,
images.Count,
snapshotId: _snapshotId,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly)
{
return new ImpactSet(
selector,
ImmutableArray<ImpactImage>.Empty,
usageOnly,
_timeProvider.GetUtcNow(),
0,
snapshotId: null,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
return new ImpactSet(
selector,
ImmutableArray<ImpactImage>.Empty,
usageOnly,
_timeProvider.GetUtcNow(),
0,
snapshotId: _snapshotId,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
private static bool ImageMatchesSelector(ImpactImageRecord image, Selector selector)
{
@@ -403,22 +525,54 @@ public sealed class RoaringImpactIndex : IImpactIndex
return RoaringBitmap.Create(remaining);
}
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
{
return selector.Scope switch
{
SelectorScope.AllImages => true,
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
{
return selector.Scope switch
{
SelectorScope.AllImages => true,
SelectorScope.ByDigest => selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase),
SelectorScope.ByRepository => selector.Repositories.Any(repo =>
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)),
SelectorScope.ByNamespace => !image.Namespaces.IsDefaultOrEmpty && selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)),
SelectorScope.ByLabels => selector.Labels.All(label =>
image.Labels.TryGetValue(label.Key, out var value) &&
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
_ => true,
};
}
SelectorScope.ByLabels => selector.Labels.All(label =>
image.Labels.TryGetValue(label.Key, out var value) &&
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
_ => true,
};
}
private static string ComputeSnapshotId(
ImmutableArray<ImpactImageRecord> images,
ImmutableDictionary<string, ImmutableArray<int>> contains,
ImmutableDictionary<string, ImmutableArray<int>> usedBy)
{
var builder = new StringBuilder();
foreach (var image in images.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase))
{
builder.Append(image.Digest).Append('|').Append(image.GeneratedAt.ToUnixTimeSeconds()).Append(';');
}
void AppendMap(ImmutableDictionary<string, ImmutableArray<int>> map)
{
foreach (var kvp in map.OrderBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase))
{
builder.Append(kvp.Key).Append('=');
foreach (var id in kvp.Value)
{
builder.Append(id).Append(',');
}
builder.Append('|');
}
}
AppendMap(contains);
AppendMap(usedBy);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
return "snap-" + Convert.ToHexString(hash).ToLowerInvariant();
}
private static bool MatchesTagPattern(string tag, string pattern)
{