feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies. - Documented roles and guidelines in AGENTS.md for Scheduler module. - Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs. - Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics. - Developed API endpoints for managing resolver jobs and retrieving metrics. - Defined models for resolver job requests and responses. - Integrated dependency injection for resolver job services. - Implemented ImpactIndexSnapshot for persisting impact index data. - Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring. - Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService. - Created dotnet-filter.sh script to handle command-line arguments for dotnet. - Established nuget-prime project for managing package downloads.
This commit is contained in:
@@ -107,21 +107,78 @@ public sealed class FixtureImpactIndex : IImpactIndex
|
||||
return CreateImpactSet(state, selector, Enumerable.Empty<FixtureMatch>(), usageOnly);
|
||||
}
|
||||
|
||||
public async ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
public async ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(selector);
|
||||
|
||||
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var matches = state.ImagesByDigest.Values
|
||||
.Select(image => new FixtureMatch(image, image.UsedByEntrypoint))
|
||||
.Where(match => !usageOnly || match.UsedByEntrypoint);
|
||||
|
||||
return CreateImpactSet(state, selector, matches, usageOnly);
|
||||
}
|
||||
var matches = state.ImagesByDigest.Values
|
||||
.Select(image => new FixtureMatch(image, image.UsedByEntrypoint))
|
||||
.Where(match => !usageOnly || match.UsedByEntrypoint);
|
||||
|
||||
return CreateImpactSet(state, selector, matches, usageOnly);
|
||||
}
|
||||
|
||||
public ValueTask RemoveAsync(string imageDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Fixture-backed index is immutable; removals are ignored.
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public async ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var images = state.ImagesByDigest.Values
|
||||
.OrderBy(image => image.Digest, StringComparer.OrdinalIgnoreCase)
|
||||
.Select((image, index) => new ImpactImageRecord(
|
||||
index,
|
||||
"fixture",
|
||||
image.Digest,
|
||||
image.Registry,
|
||||
image.Repository,
|
||||
image.Namespaces,
|
||||
image.Tags,
|
||||
image.Labels,
|
||||
image.GeneratedAt,
|
||||
image.Components.Select(c => c.Purl).ToImmutableArray(),
|
||||
image.Components.Where(c => c.UsedByEntrypoint).Select(c => c.Purl).ToImmutableArray()))
|
||||
.ToImmutableArray();
|
||||
|
||||
var contains = images
|
||||
.SelectMany(img => img.Components.Select(purl => (purl, img.ImageId)))
|
||||
.GroupBy(pair => pair.purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableDictionary(
|
||||
g => g.Key,
|
||||
g => g.Select(p => p.ImageId).Distinct().OrderBy(id => id).ToImmutableArray(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var usedBy = images
|
||||
.SelectMany(img => img.EntrypointComponents.Select(purl => (purl, img.ImageId)))
|
||||
.GroupBy(pair => pair.purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableDictionary(
|
||||
g => g.Key,
|
||||
g => g.Select(p => p.ImageId).Distinct().OrderBy(id => id).ToImmutableArray(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
return new ImpactIndexSnapshot(
|
||||
state.GeneratedAt,
|
||||
state.SnapshotId,
|
||||
images,
|
||||
contains,
|
||||
usedBy);
|
||||
}
|
||||
|
||||
public ValueTask RestoreSnapshotAsync(ImpactIndexSnapshot snapshot, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
// Fixture index remains immutable; restoration is a no-op.
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task<FixtureIndexState> EnsureInitializedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -39,8 +39,29 @@ public interface IImpactIndex
|
||||
/// <param name="selector">Selector scoping the query.</param>
|
||||
/// <param name="usageOnly">When true, restricts results to images with entrypoint usage.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Removes an image digest and its component mappings from the index.
|
||||
/// Used when an image is deleted or aged out.
|
||||
/// </summary>
|
||||
ValueTask RemoveAsync(
|
||||
string imageDigest,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a compacted snapshot of the index for persistence (e.g., RocksDB/Redis).
|
||||
/// </summary>
|
||||
ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Restores index state from a previously persisted snapshot.
|
||||
/// </summary>
|
||||
ValueTask RestoreSnapshotAsync(
|
||||
ImpactIndexSnapshot snapshot,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
internal sealed record ImpactImageRecord(
|
||||
public sealed record ImpactImageRecord(
|
||||
int ImageId,
|
||||
string TenantId,
|
||||
string Digest,
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Serializable snapshot for persisting the ImpactIndex (e.g., RocksDB/Redis).
|
||||
/// Contains compacted image IDs and per-purl bitmap membership.
|
||||
/// </summary>
|
||||
public sealed record ImpactIndexSnapshot(
|
||||
DateTimeOffset GeneratedAt,
|
||||
string SnapshotId,
|
||||
ImmutableArray<ImpactImageRecord> Images,
|
||||
ImmutableDictionary<string, ImmutableArray<int>> ContainsByPurl,
|
||||
ImmutableDictionary<string, ImmutableArray<int>> UsedByEntrypointByPurl)
|
||||
{
|
||||
public static byte[] ToBytes(ImpactIndexSnapshot snapshot)
|
||||
{
|
||||
var options = SerializerOptions;
|
||||
return JsonSerializer.SerializeToUtf8Bytes(snapshot, options);
|
||||
}
|
||||
|
||||
public static ImpactIndexSnapshot FromBytes(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var options = SerializerOptions;
|
||||
var snapshot = JsonSerializer.Deserialize<ImpactIndexSnapshot>(payload, options);
|
||||
return snapshot ?? throw new InvalidOperationException("ImpactIndexSnapshot payload could not be deserialized.");
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
}
|
||||
@@ -23,16 +23,41 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
private readonly Dictionary<string, int> _imageIds = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<int, ImpactImageRecord> _images = new();
|
||||
private readonly Dictionary<string, RoaringBitmap> _containsByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private readonly ILogger<RoaringImpactIndex> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private string? _snapshotId;
|
||||
|
||||
private readonly ILogger<RoaringImpactIndex> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public ValueTask RemoveAsync(string imageDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_imageIds.TryGetValue(imageDigest, out var imageId))
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
if (_images.TryGetValue(imageId, out var record))
|
||||
{
|
||||
RemoveImageComponents(record);
|
||||
_images.Remove(imageId);
|
||||
}
|
||||
|
||||
_imageIds.Remove(imageDigest);
|
||||
_snapshotId = null;
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task IngestAsync(ImpactIndexIngestionRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -130,11 +155,108 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(CreateEmptyImpactSet(selector, usageOnly));
|
||||
|
||||
public ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
|
||||
public ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
|
||||
|
||||
public ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
var orderedImages = _images
|
||||
.Values
|
||||
.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(img => img.Repository, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
var idMap = orderedImages
|
||||
.Select((image, index) => (image.ImageId, NewId: index))
|
||||
.ToDictionary(tuple => tuple.ImageId, tuple => tuple.NewId);
|
||||
|
||||
var compactedImages = orderedImages
|
||||
.Select(image => image with { ImageId = idMap[image.ImageId] })
|
||||
.ToImmutableArray();
|
||||
|
||||
ImmutableDictionary<string, ImmutableArray<int>> CompactBitmaps(Dictionary<string, RoaringBitmap> source)
|
||||
{
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, ImmutableArray<int>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var (key, bitmap) in source)
|
||||
{
|
||||
var remapped = bitmap
|
||||
.Select(id => idMap.TryGetValue(id, out var newId) ? newId : (int?)null)
|
||||
.Where(id => id.HasValue)
|
||||
.Select(id => id!.Value)
|
||||
.Distinct()
|
||||
.OrderBy(id => id)
|
||||
.ToImmutableArray();
|
||||
|
||||
if (remapped.Length > 0)
|
||||
{
|
||||
builder[key] = remapped;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
var contains = CompactBitmaps(_containsByPurl);
|
||||
var usedBy = CompactBitmaps(_usedByEntrypointByPurl);
|
||||
|
||||
var generatedAt = orderedImages.Length == 0
|
||||
? _timeProvider.GetUtcNow()
|
||||
: orderedImages.Max(img => img.GeneratedAt);
|
||||
|
||||
var snapshotId = ComputeSnapshotId(compactedImages, contains, usedBy);
|
||||
_snapshotId = snapshotId;
|
||||
|
||||
var snapshot = new ImpactIndexSnapshot(
|
||||
generatedAt,
|
||||
snapshotId,
|
||||
compactedImages,
|
||||
contains,
|
||||
usedBy);
|
||||
|
||||
return ValueTask.FromResult(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask RestoreSnapshotAsync(ImpactIndexSnapshot snapshot, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_images.Clear();
|
||||
_imageIds.Clear();
|
||||
_containsByPurl.Clear();
|
||||
_usedByEntrypointByPurl.Clear();
|
||||
|
||||
foreach (var image in snapshot.Images)
|
||||
{
|
||||
_images[image.ImageId] = image;
|
||||
_imageIds[image.Digest] = image.ImageId;
|
||||
}
|
||||
|
||||
foreach (var kvp in snapshot.ContainsByPurl)
|
||||
{
|
||||
_containsByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
|
||||
}
|
||||
|
||||
foreach (var kvp in snapshot.UsedByEntrypointByPurl)
|
||||
{
|
||||
_usedByEntrypointByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
|
||||
}
|
||||
|
||||
_snapshotId = snapshot.SnapshotId;
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private ImpactSet ResolveByPurlsCore(IEnumerable<string> purls, bool usageOnly, Selector selector)
|
||||
{
|
||||
@@ -231,27 +353,27 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
|
||||
var generatedAt = latestGeneratedAt == DateTimeOffset.MinValue ? _timeProvider.GetUtcNow() : latestGeneratedAt;
|
||||
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
images.ToImmutableArray(),
|
||||
usageOnly,
|
||||
generatedAt,
|
||||
images.Count,
|
||||
snapshotId: null,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
images.ToImmutableArray(),
|
||||
usageOnly,
|
||||
generatedAt,
|
||||
images.Count,
|
||||
snapshotId: _snapshotId,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly)
|
||||
{
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
ImmutableArray<ImpactImage>.Empty,
|
||||
usageOnly,
|
||||
_timeProvider.GetUtcNow(),
|
||||
0,
|
||||
snapshotId: null,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
ImmutableArray<ImpactImage>.Empty,
|
||||
usageOnly,
|
||||
_timeProvider.GetUtcNow(),
|
||||
0,
|
||||
snapshotId: _snapshotId,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private static bool ImageMatchesSelector(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
@@ -403,22 +525,54 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
return RoaringBitmap.Create(remaining);
|
||||
}
|
||||
|
||||
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
return selector.Scope switch
|
||||
{
|
||||
SelectorScope.AllImages => true,
|
||||
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
return selector.Scope switch
|
||||
{
|
||||
SelectorScope.AllImages => true,
|
||||
SelectorScope.ByDigest => selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase),
|
||||
SelectorScope.ByRepository => selector.Repositories.Any(repo =>
|
||||
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByNamespace => !image.Namespaces.IsDefaultOrEmpty && selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByLabels => selector.Labels.All(label =>
|
||||
image.Labels.TryGetValue(label.Key, out var value) &&
|
||||
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
SelectorScope.ByLabels => selector.Labels.All(label =>
|
||||
image.Labels.TryGetValue(label.Key, out var value) &&
|
||||
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
|
||||
private static string ComputeSnapshotId(
|
||||
ImmutableArray<ImpactImageRecord> images,
|
||||
ImmutableDictionary<string, ImmutableArray<int>> contains,
|
||||
ImmutableDictionary<string, ImmutableArray<int>> usedBy)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
|
||||
foreach (var image in images.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
builder.Append(image.Digest).Append('|').Append(image.GeneratedAt.ToUnixTimeSeconds()).Append(';');
|
||||
}
|
||||
|
||||
void AppendMap(ImmutableDictionary<string, ImmutableArray<int>> map)
|
||||
{
|
||||
foreach (var kvp in map.OrderBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
builder.Append(kvp.Key).Append('=');
|
||||
foreach (var id in kvp.Value)
|
||||
{
|
||||
builder.Append(id).Append(',');
|
||||
}
|
||||
builder.Append('|');
|
||||
}
|
||||
}
|
||||
|
||||
AppendMap(contains);
|
||||
AppendMap(usedBy);
|
||||
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
|
||||
return "snap-" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static bool MatchesTagPattern(string tag, string pattern)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user