feat: Add initial implementation of Vulnerability Resolver Jobs
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Created project for StellaOps.Scanner.Analyzers.Native.Tests with necessary dependencies. - Documented roles and guidelines in AGENTS.md for Scheduler module. - Implemented IResolverJobService interface and InMemoryResolverJobService for handling resolver jobs. - Added ResolverBacklogNotifier and ResolverBacklogService for monitoring job metrics. - Developed API endpoints for managing resolver jobs and retrieving metrics. - Defined models for resolver job requests and responses. - Integrated dependency injection for resolver job services. - Implemented ImpactIndexSnapshot for persisting impact index data. - Introduced SignalsScoringOptions for configurable scoring weights in reachability scoring. - Added unit tests for ReachabilityScoringService and RuntimeFactsIngestionService. - Created dotnet-filter.sh script to handle command-line arguments for dotnet. - Established nuget-prime project for managing package downloads.
This commit is contained in:
master
@@ -23,16 +23,41 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
private readonly Dictionary<string, int> _imageIds = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<int, ImpactImageRecord> _images = new();
|
||||
private readonly Dictionary<string, RoaringBitmap> _containsByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private readonly ILogger<RoaringImpactIndex> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private string? _snapshotId;
|
||||
|
||||
private readonly ILogger<RoaringImpactIndex> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public ValueTask RemoveAsync(string imageDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_imageIds.TryGetValue(imageDigest, out var imageId))
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
if (_images.TryGetValue(imageId, out var record))
|
||||
{
|
||||
RemoveImageComponents(record);
|
||||
_images.Remove(imageId);
|
||||
}
|
||||
|
||||
_imageIds.Remove(imageDigest);
|
||||
_snapshotId = null;
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task IngestAsync(ImpactIndexIngestionRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -130,11 +155,108 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(CreateEmptyImpactSet(selector, usageOnly));
|
||||
|
||||
public ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
|
||||
public ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
|
||||
|
||||
public ValueTask<ImpactIndexSnapshot> CreateSnapshotAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
var orderedImages = _images
|
||||
.Values
|
||||
.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(img => img.Repository, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
var idMap = orderedImages
|
||||
.Select((image, index) => (image.ImageId, NewId: index))
|
||||
.ToDictionary(tuple => tuple.ImageId, tuple => tuple.NewId);
|
||||
|
||||
var compactedImages = orderedImages
|
||||
.Select(image => image with { ImageId = idMap[image.ImageId] })
|
||||
.ToImmutableArray();
|
||||
|
||||
ImmutableDictionary<string, ImmutableArray<int>> CompactBitmaps(Dictionary<string, RoaringBitmap> source)
|
||||
{
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, ImmutableArray<int>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var (key, bitmap) in source)
|
||||
{
|
||||
var remapped = bitmap
|
||||
.Select(id => idMap.TryGetValue(id, out var newId) ? newId : (int?)null)
|
||||
.Where(id => id.HasValue)
|
||||
.Select(id => id!.Value)
|
||||
.Distinct()
|
||||
.OrderBy(id => id)
|
||||
.ToImmutableArray();
|
||||
|
||||
if (remapped.Length > 0)
|
||||
{
|
||||
builder[key] = remapped;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
var contains = CompactBitmaps(_containsByPurl);
|
||||
var usedBy = CompactBitmaps(_usedByEntrypointByPurl);
|
||||
|
||||
var generatedAt = orderedImages.Length == 0
|
||||
? _timeProvider.GetUtcNow()
|
||||
: orderedImages.Max(img => img.GeneratedAt);
|
||||
|
||||
var snapshotId = ComputeSnapshotId(compactedImages, contains, usedBy);
|
||||
_snapshotId = snapshotId;
|
||||
|
||||
var snapshot = new ImpactIndexSnapshot(
|
||||
generatedAt,
|
||||
snapshotId,
|
||||
compactedImages,
|
||||
contains,
|
||||
usedBy);
|
||||
|
||||
return ValueTask.FromResult(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask RestoreSnapshotAsync(ImpactIndexSnapshot snapshot, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_images.Clear();
|
||||
_imageIds.Clear();
|
||||
_containsByPurl.Clear();
|
||||
_usedByEntrypointByPurl.Clear();
|
||||
|
||||
foreach (var image in snapshot.Images)
|
||||
{
|
||||
_images[image.ImageId] = image;
|
||||
_imageIds[image.Digest] = image.ImageId;
|
||||
}
|
||||
|
||||
foreach (var kvp in snapshot.ContainsByPurl)
|
||||
{
|
||||
_containsByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
|
||||
}
|
||||
|
||||
foreach (var kvp in snapshot.UsedByEntrypointByPurl)
|
||||
{
|
||||
_usedByEntrypointByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
|
||||
}
|
||||
|
||||
_snapshotId = snapshot.SnapshotId;
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private ImpactSet ResolveByPurlsCore(IEnumerable<string> purls, bool usageOnly, Selector selector)
|
||||
{
|
||||
@@ -231,27 +353,27 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
|
||||
var generatedAt = latestGeneratedAt == DateTimeOffset.MinValue ? _timeProvider.GetUtcNow() : latestGeneratedAt;
|
||||
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
images.ToImmutableArray(),
|
||||
usageOnly,
|
||||
generatedAt,
|
||||
images.Count,
|
||||
snapshotId: null,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
images.ToImmutableArray(),
|
||||
usageOnly,
|
||||
generatedAt,
|
||||
images.Count,
|
||||
snapshotId: _snapshotId,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly)
|
||||
{
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
ImmutableArray<ImpactImage>.Empty,
|
||||
usageOnly,
|
||||
_timeProvider.GetUtcNow(),
|
||||
0,
|
||||
snapshotId: null,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
ImmutableArray<ImpactImage>.Empty,
|
||||
usageOnly,
|
||||
_timeProvider.GetUtcNow(),
|
||||
0,
|
||||
snapshotId: _snapshotId,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private static bool ImageMatchesSelector(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
@@ -403,22 +525,54 @@ public sealed class RoaringImpactIndex : IImpactIndex
|
||||
return RoaringBitmap.Create(remaining);
|
||||
}
|
||||
|
||||
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
return selector.Scope switch
|
||||
{
|
||||
SelectorScope.AllImages => true,
|
||||
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
return selector.Scope switch
|
||||
{
|
||||
SelectorScope.AllImages => true,
|
||||
SelectorScope.ByDigest => selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase),
|
||||
SelectorScope.ByRepository => selector.Repositories.Any(repo =>
|
||||
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByNamespace => !image.Namespaces.IsDefaultOrEmpty && selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByLabels => selector.Labels.All(label =>
|
||||
image.Labels.TryGetValue(label.Key, out var value) &&
|
||||
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
SelectorScope.ByLabels => selector.Labels.All(label =>
|
||||
image.Labels.TryGetValue(label.Key, out var value) &&
|
||||
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
|
||||
private static string ComputeSnapshotId(
|
||||
ImmutableArray<ImpactImageRecord> images,
|
||||
ImmutableDictionary<string, ImmutableArray<int>> contains,
|
||||
ImmutableDictionary<string, ImmutableArray<int>> usedBy)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
|
||||
foreach (var image in images.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
builder.Append(image.Digest).Append('|').Append(image.GeneratedAt.ToUnixTimeSeconds()).Append(';');
|
||||
}
|
||||
|
||||
void AppendMap(ImmutableDictionary<string, ImmutableArray<int>> map)
|
||||
{
|
||||
foreach (var kvp in map.OrderBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
builder.Append(kvp.Key).Append('=');
|
||||
foreach (var id in kvp.Value)
|
||||
{
|
||||
builder.Append(id).Append(',');
|
||||
}
|
||||
builder.Append('|');
|
||||
}
|
||||
}
|
||||
|
||||
AppendMap(contains);
|
||||
AppendMap(usedBy);
|
||||
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
|
||||
return "snap-" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static bool MatchesTagPattern(string tag, string pattern)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user