Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,4 @@
|
||||
# StellaOps.Scheduler.ImpactIndex — Agent Charter
|
||||
|
||||
## Mission
|
||||
Build the global impact index per `docs/ARCHITECTURE_SCHEDULER.md` (roaring bitmaps, selectors, snapshotting).
|
||||
@@ -0,0 +1,615 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.IO;
|
||||
using System.IO.Enumeration;
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scheduler.Models;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Fixture-backed implementation of <see cref="IImpactIndex"/> used while the real index is under construction.
|
||||
/// </summary>
|
||||
public sealed class FixtureImpactIndex : IImpactIndex
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
};
|
||||
|
||||
private readonly ImpactIndexStubOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<FixtureImpactIndex> _logger;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private FixtureIndexState? _state;
|
||||
|
||||
public FixtureImpactIndex(
|
||||
ImpactIndexStubOptions options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<FixtureImpactIndex> logger)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async ValueTask<ImpactSet> ResolveByPurlsAsync(
|
||||
IEnumerable<string> purls,
|
||||
bool usageOnly,
|
||||
Selector selector,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(purls);
|
||||
ArgumentNullException.ThrowIfNull(selector);
|
||||
|
||||
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var normalizedPurls = NormalizeKeys(purls);
|
||||
|
||||
if (normalizedPurls.Length == 0)
|
||||
{
|
||||
return CreateImpactSet(state, selector, Enumerable.Empty<FixtureMatch>(), usageOnly);
|
||||
}
|
||||
|
||||
var matches = new List<FixtureMatch>();
|
||||
foreach (var purl in normalizedPurls)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!state.PurlIndex.TryGetValue(purl, out var componentMatches))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var component in componentMatches)
|
||||
{
|
||||
var usedByEntrypoint = component.Component.UsedByEntrypoint;
|
||||
if (usageOnly && !usedByEntrypoint)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
matches.Add(new FixtureMatch(component.Image, usedByEntrypoint));
|
||||
}
|
||||
}
|
||||
|
||||
return CreateImpactSet(state, selector, matches, usageOnly);
|
||||
}
|
||||
|
||||
public async ValueTask<ImpactSet> ResolveByVulnerabilitiesAsync(
|
||||
IEnumerable<string> vulnerabilityIds,
|
||||
bool usageOnly,
|
||||
Selector selector,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(vulnerabilityIds);
|
||||
ArgumentNullException.ThrowIfNull(selector);
|
||||
|
||||
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// The stub does not maintain a vulnerability → purl projection, so we return an empty result.
|
||||
if (_logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
var first = vulnerabilityIds.FirstOrDefault(static id => !string.IsNullOrWhiteSpace(id));
|
||||
if (first is not null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"ImpactIndex stub received ResolveByVulnerabilitiesAsync for '{VulnerabilityId}' but mappings are not available.",
|
||||
first);
|
||||
}
|
||||
}
|
||||
|
||||
return CreateImpactSet(state, selector, Enumerable.Empty<FixtureMatch>(), usageOnly);
|
||||
}
|
||||
|
||||
public async ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(selector);
|
||||
|
||||
var state = await EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var matches = state.ImagesByDigest.Values
|
||||
.Select(image => new FixtureMatch(image, image.UsedByEntrypoint))
|
||||
.Where(match => !usageOnly || match.UsedByEntrypoint);
|
||||
|
||||
return CreateImpactSet(state, selector, matches, usageOnly);
|
||||
}
|
||||
|
||||
private async Task<FixtureIndexState> EnsureInitializedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_state is not null)
|
||||
{
|
||||
return _state;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_state is not null)
|
||||
{
|
||||
return _state;
|
||||
}
|
||||
|
||||
var state = await LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
_state = state;
|
||||
_logger.LogInformation(
|
||||
"ImpactIndex stub loaded {ImageCount} fixture images from {SourceDescription}.",
|
||||
state.ImagesByDigest.Count,
|
||||
state.SourceDescription);
|
||||
return state;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<FixtureIndexState> LoadAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var images = new List<FixtureImage>();
|
||||
string? sourceDescription = null;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_options.FixtureDirectory))
|
||||
{
|
||||
var directory = ResolveDirectoryPath(_options.FixtureDirectory!);
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
images.AddRange(await LoadFromDirectoryAsync(directory, cancellationToken).ConfigureAwait(false));
|
||||
sourceDescription = directory;
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"ImpactIndex stub fixture directory '{Directory}' was not found. Falling back to embedded fixtures.",
|
||||
directory);
|
||||
}
|
||||
}
|
||||
|
||||
if (images.Count == 0)
|
||||
{
|
||||
images.AddRange(await LoadFromResourcesAsync(cancellationToken).ConfigureAwait(false));
|
||||
sourceDescription ??= "embedded:scheduler-impact-index-fixtures";
|
||||
}
|
||||
|
||||
if (images.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("No BOM-Index fixtures were found for the ImpactIndex stub.");
|
||||
}
|
||||
|
||||
return BuildState(images, sourceDescription!, _options.SnapshotId);
|
||||
}
|
||||
|
||||
private static string ResolveDirectoryPath(string path)
|
||||
{
|
||||
if (Path.IsPathRooted(path))
|
||||
{
|
||||
return path;
|
||||
}
|
||||
|
||||
var basePath = AppContext.BaseDirectory;
|
||||
return Path.GetFullPath(Path.Combine(basePath, path));
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<FixtureImage>> LoadFromDirectoryAsync(
|
||||
string directory,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<FixtureImage>();
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(directory, "bom-index.json", SearchOption.AllDirectories)
|
||||
.OrderBy(static file => file, StringComparer.Ordinal))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
await using var stream = File.OpenRead(file);
|
||||
var document = await JsonSerializer.DeserializeAsync<BomIndexDocument>(stream, SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
results.Add(CreateFixtureImage(document));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<FixtureImage>> LoadFromResourcesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var assembly = typeof(FixtureImpactIndex).Assembly;
|
||||
var resourceNames = assembly
|
||||
.GetManifestResourceNames()
|
||||
.Where(static name => name.EndsWith(".bom-index.json", StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(static name => name, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var results = new List<FixtureImage>(resourceNames.Length);
|
||||
|
||||
foreach (var resourceName in resourceNames)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
await using var stream = assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var document = await JsonSerializer.DeserializeAsync<BomIndexDocument>(stream, SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
results.Add(CreateFixtureImage(document));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static FixtureIndexState BuildState(
|
||||
IReadOnlyList<FixtureImage> images,
|
||||
string sourceDescription,
|
||||
string snapshotId)
|
||||
{
|
||||
var imagesByDigest = images
|
||||
.GroupBy(static image => image.Digest, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableDictionary(
|
||||
static group => group.Key,
|
||||
static group => group
|
||||
.OrderBy(static image => image.Repository, StringComparer.Ordinal)
|
||||
.ThenBy(static image => image.Registry, StringComparer.Ordinal)
|
||||
.ThenBy(static image => image.Tags.Length, Comparer<int>.Default)
|
||||
.First(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var purlIndexBuilder = new Dictionary<string, List<FixtureComponentMatch>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var image in images)
|
||||
{
|
||||
foreach (var component in image.Components)
|
||||
{
|
||||
if (!purlIndexBuilder.TryGetValue(component.Purl, out var list))
|
||||
{
|
||||
list = new List<FixtureComponentMatch>();
|
||||
purlIndexBuilder[component.Purl] = list;
|
||||
}
|
||||
|
||||
list.Add(new FixtureComponentMatch(image, component));
|
||||
}
|
||||
}
|
||||
|
||||
var purlIndex = purlIndexBuilder.ToImmutableDictionary(
|
||||
static entry => entry.Key,
|
||||
static entry => entry.Value
|
||||
.OrderBy(static item => item.Image.Digest, StringComparer.Ordinal)
|
||||
.Select(static item => new FixtureComponentMatch(item.Image, item.Component))
|
||||
.ToImmutableArray(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var generatedAt = images.Count == 0
|
||||
? DateTimeOffset.UnixEpoch
|
||||
: images.Max(static image => image.GeneratedAt);
|
||||
|
||||
return new FixtureIndexState(imagesByDigest, purlIndex, generatedAt, sourceDescription, snapshotId);
|
||||
}
|
||||
|
||||
private ImpactSet CreateImpactSet(
|
||||
FixtureIndexState state,
|
||||
Selector selector,
|
||||
IEnumerable<FixtureMatch> matches,
|
||||
bool usageOnly)
|
||||
{
|
||||
var aggregated = new Dictionary<string, ImpactImageBuilder>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var match in matches)
|
||||
{
|
||||
if (!ImageMatchesSelector(match.Image, selector))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!aggregated.TryGetValue(match.Image.Digest, out var builder))
|
||||
{
|
||||
builder = new ImpactImageBuilder(match.Image);
|
||||
aggregated[match.Image.Digest] = builder;
|
||||
}
|
||||
|
||||
builder.MarkUsedByEntrypoint(match.UsedByEntrypoint);
|
||||
}
|
||||
|
||||
var images = aggregated.Values
|
||||
.Select(static builder => builder.Build())
|
||||
.OrderBy(static image => image.ImageDigest, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
images,
|
||||
usageOnly,
|
||||
state.GeneratedAt == DateTimeOffset.UnixEpoch
|
||||
? _timeProvider.GetUtcNow()
|
||||
: state.GeneratedAt,
|
||||
images.Length,
|
||||
state.SnapshotId,
|
||||
SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private static bool ImageMatchesSelector(FixtureImage image, Selector selector)
|
||||
{
|
||||
if (selector is null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (selector.Digests.Length > 0 &&
|
||||
!selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (selector.Repositories.Length > 0)
|
||||
{
|
||||
var repositoryMatch = selector.Repositories.Any(repo =>
|
||||
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (!repositoryMatch)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.Namespaces.Length > 0)
|
||||
{
|
||||
if (image.Namespaces.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var namespaceMatch = selector.Namespaces.Any(namespaceId =>
|
||||
image.Namespaces.Contains(namespaceId, StringComparer.OrdinalIgnoreCase));
|
||||
|
||||
if (!namespaceMatch)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.IncludeTags.Length > 0)
|
||||
{
|
||||
if (image.Tags.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var tagMatch = selector.IncludeTags.Any(pattern =>
|
||||
MatchesAnyTag(image.Tags, pattern));
|
||||
|
||||
if (!tagMatch)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.Labels.Length > 0)
|
||||
{
|
||||
if (image.Labels.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var labelSelector in selector.Labels)
|
||||
{
|
||||
if (!image.Labels.TryGetValue(labelSelector.Key, out var value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (labelSelector.Values.Length > 0 &&
|
||||
!labelSelector.Values.Contains(value, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return selector.Scope switch
|
||||
{
|
||||
SelectorScope.ByDigest => selector.Digests.Length == 0
|
||||
? true
|
||||
: selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase),
|
||||
SelectorScope.ByRepository => selector.Repositories.Length == 0
|
||||
? true
|
||||
: selector.Repositories.Any(repo =>
|
||||
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByNamespace => selector.Namespaces.Length == 0
|
||||
? true
|
||||
: !image.Namespaces.IsDefaultOrEmpty &&
|
||||
selector.Namespaces.Any(namespaceId =>
|
||||
image.Namespaces.Contains(namespaceId, StringComparer.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByLabels => selector.Labels.Length == 0
|
||||
? true
|
||||
: selector.Labels.All(label =>
|
||||
image.Labels.TryGetValue(label.Key, out var value) &&
|
||||
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
|
||||
private static bool MatchesAnyTag(ImmutableArray<string> tags, string pattern)
|
||||
{
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
if (FileSystemName.MatchesSimpleExpression(pattern, tag, ignoreCase: true))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static FixtureImage CreateFixtureImage(BomIndexDocument document)
|
||||
{
|
||||
if (document.Image is null)
|
||||
{
|
||||
throw new InvalidOperationException("BOM-Index image metadata is required.");
|
||||
}
|
||||
|
||||
var digest = Validation.EnsureDigestFormat(document.Image.Digest, "image.digest");
|
||||
var (registry, repository) = SplitRepository(document.Image.Repository);
|
||||
|
||||
var tags = string.IsNullOrWhiteSpace(document.Image.Tag)
|
||||
? ImmutableArray<string>.Empty
|
||||
: ImmutableArray.Create(document.Image.Tag.Trim());
|
||||
|
||||
var components = (document.Components ?? Array.Empty<BomIndexComponent>())
|
||||
.Where(static component => !string.IsNullOrWhiteSpace(component.Purl))
|
||||
.Select(component => new FixtureComponent(
|
||||
component.Purl!.Trim(),
|
||||
component.Usage?.Any(static usage =>
|
||||
usage.Equals("runtime", StringComparison.OrdinalIgnoreCase) ||
|
||||
usage.Equals("usedByEntrypoint", StringComparison.OrdinalIgnoreCase)) == true))
|
||||
.OrderBy(static component => component.Purl, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new FixtureImage(
|
||||
digest,
|
||||
registry,
|
||||
repository,
|
||||
ImmutableArray<string>.Empty,
|
||||
tags,
|
||||
ImmutableSortedDictionary<string, string>.Empty.WithComparers(StringComparer.OrdinalIgnoreCase),
|
||||
components,
|
||||
document.GeneratedAt == default ? DateTimeOffset.UnixEpoch : document.GeneratedAt.ToUniversalTime(),
|
||||
components.Any(static component => component.UsedByEntrypoint));
|
||||
}
|
||||
|
||||
private static (string Registry, string Repository) SplitRepository(string repository)
|
||||
{
|
||||
var normalized = Validation.EnsureNotNullOrWhiteSpace(repository, nameof(repository));
|
||||
var separatorIndex = normalized.IndexOf('/');
|
||||
if (separatorIndex < 0)
|
||||
{
|
||||
return ("docker.io", normalized);
|
||||
}
|
||||
|
||||
var registry = normalized[..separatorIndex];
|
||||
var repo = normalized[(separatorIndex + 1)..];
|
||||
if (string.IsNullOrWhiteSpace(repo))
|
||||
{
|
||||
throw new ArgumentException("Repository segment is required after registry.", nameof(repository));
|
||||
}
|
||||
|
||||
return (registry.Trim(), repo.Trim());
|
||||
}
|
||||
|
||||
private static string[] NormalizeKeys(IEnumerable<string> values)
|
||||
{
|
||||
return values
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private readonly record struct FixtureMatch(FixtureImage Image, bool UsedByEntrypoint);
|
||||
|
||||
private sealed record FixtureImage(
|
||||
string Digest,
|
||||
string Registry,
|
||||
string Repository,
|
||||
ImmutableArray<string> Namespaces,
|
||||
ImmutableArray<string> Tags,
|
||||
ImmutableSortedDictionary<string, string> Labels,
|
||||
ImmutableArray<FixtureComponent> Components,
|
||||
DateTimeOffset GeneratedAt,
|
||||
bool UsedByEntrypoint);
|
||||
|
||||
private sealed record FixtureComponent(string Purl, bool UsedByEntrypoint);
|
||||
|
||||
private sealed record FixtureComponentMatch(FixtureImage Image, FixtureComponent Component);
|
||||
|
||||
private sealed record FixtureIndexState(
|
||||
ImmutableDictionary<string, FixtureImage> ImagesByDigest,
|
||||
ImmutableDictionary<string, ImmutableArray<FixtureComponentMatch>> PurlIndex,
|
||||
DateTimeOffset GeneratedAt,
|
||||
string SourceDescription,
|
||||
string SnapshotId);
|
||||
|
||||
private sealed class ImpactImageBuilder
|
||||
{
|
||||
private readonly FixtureImage _image;
|
||||
private bool _usedByEntrypoint;
|
||||
|
||||
public ImpactImageBuilder(FixtureImage image)
|
||||
{
|
||||
_image = image;
|
||||
}
|
||||
|
||||
public void MarkUsedByEntrypoint(bool usedByEntrypoint)
|
||||
{
|
||||
_usedByEntrypoint |= usedByEntrypoint;
|
||||
}
|
||||
|
||||
public ImpactImage Build()
|
||||
{
|
||||
return new ImpactImage(
|
||||
_image.Digest,
|
||||
_image.Registry,
|
||||
_image.Repository,
|
||||
_image.Namespaces,
|
||||
_image.Tags,
|
||||
_usedByEntrypoint,
|
||||
_image.Labels);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record BomIndexDocument
|
||||
{
|
||||
[JsonPropertyName("schema")]
|
||||
public string? Schema { get; init; }
|
||||
|
||||
[JsonPropertyName("image")]
|
||||
public BomIndexImage? Image { get; init; }
|
||||
|
||||
[JsonPropertyName("generatedAt")]
|
||||
public DateTimeOffset GeneratedAt { get; init; }
|
||||
|
||||
[JsonPropertyName("components")]
|
||||
public IReadOnlyList<BomIndexComponent>? Components { get; init; }
|
||||
}
|
||||
|
||||
private sealed record BomIndexImage
|
||||
{
|
||||
[JsonPropertyName("repository")]
|
||||
public string Repository { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("digest")]
|
||||
public string Digest { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("tag")]
|
||||
public string? Tag { get; init; }
|
||||
}
|
||||
|
||||
private sealed record BomIndexComponent
|
||||
{
|
||||
[JsonPropertyName("purl")]
|
||||
public string? Purl { get; init; }
|
||||
|
||||
[JsonPropertyName("usage")]
|
||||
public IReadOnlyList<string>? Usage { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using StellaOps.Scheduler.Models;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Provides read access to the scheduler impact index.
|
||||
/// </summary>
|
||||
public interface IImpactIndex
|
||||
{
|
||||
/// <summary>
|
||||
/// Resolves the impacted image set for the provided package URLs.
|
||||
/// </summary>
|
||||
/// <param name="purls">Package URLs to look up.</param>
|
||||
/// <param name="usageOnly">When true, restricts results to components marked as runtime/entrypoint usage.</param>
|
||||
/// <param name="selector">Selector scoping the query.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
ValueTask<ImpactSet> ResolveByPurlsAsync(
|
||||
IEnumerable<string> purls,
|
||||
bool usageOnly,
|
||||
Selector selector,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Resolves impacted images by vulnerability identifiers if the index has the mapping available.
|
||||
/// </summary>
|
||||
/// <param name="vulnerabilityIds">Vulnerability identifiers to look up.</param>
|
||||
/// <param name="usageOnly">When true, restricts results to components marked as runtime/entrypoint usage.</param>
|
||||
/// <param name="selector">Selector scoping the query.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
ValueTask<ImpactSet> ResolveByVulnerabilitiesAsync(
|
||||
IEnumerable<string> vulnerabilityIds,
|
||||
bool usageOnly,
|
||||
Selector selector,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Resolves all tracked images for the provided selector.
|
||||
/// </summary>
|
||||
/// <param name="selector">Selector scoping the query.</param>
|
||||
/// <param name="usageOnly">When true, restricts results to images with entrypoint usage.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
internal sealed record ImpactImageRecord(
|
||||
int ImageId,
|
||||
string TenantId,
|
||||
string Digest,
|
||||
string Registry,
|
||||
string Repository,
|
||||
ImmutableArray<string> Namespaces,
|
||||
ImmutableArray<string> Tags,
|
||||
ImmutableSortedDictionary<string, string> Labels,
|
||||
DateTimeOffset GeneratedAt,
|
||||
ImmutableArray<string> Components,
|
||||
ImmutableArray<string> EntrypointComponents);
|
||||
@@ -0,0 +1,26 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
/// <summary>
|
||||
/// ServiceCollection helpers for wiring the fixture-backed impact index.
|
||||
/// </summary>
|
||||
public static class ImpactIndexServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddImpactIndexStub(
|
||||
this IServiceCollection services,
|
||||
Action<ImpactIndexStubOptions>? configure = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var options = new ImpactIndexStubOptions();
|
||||
configure?.Invoke(options);
|
||||
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
services.AddSingleton(options);
|
||||
services.TryAddSingleton<IImpactIndex, FixtureImpactIndex>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Options controlling the fixture-backed impact index stub.
|
||||
/// </summary>
|
||||
public sealed class ImpactIndexStubOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Optional absolute or relative directory containing BOM-Index JSON fixtures.
|
||||
/// When not supplied or not found, embedded fixtures ship with the assembly are used instead.
|
||||
/// </summary>
|
||||
public string? FixtureDirectory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot identifier reported in the generated <see cref="StellaOps.Scheduler.Models.ImpactSet"/>.
|
||||
/// Defaults to <c>samples/impact-index-stub</c>.
|
||||
/// </summary>
|
||||
public string SnapshotId { get; set; } = "samples/impact-index-stub";
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Collections.Special;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex.Ingestion;
|
||||
|
||||
internal sealed record BomIndexComponent(string Key, bool UsedByEntrypoint);
|
||||
|
||||
internal sealed record BomIndexDocument(string ImageDigest, DateTimeOffset GeneratedAt, ImmutableArray<BomIndexComponent> Components);
|
||||
|
||||
internal static class BomIndexReader
|
||||
{
|
||||
private const int HeaderMagicLength = 7;
|
||||
private static readonly byte[] Magic = Encoding.ASCII.GetBytes("BOMIDX1");
|
||||
|
||||
public static BomIndexDocument Read(Stream stream)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
|
||||
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true);
|
||||
Span<byte> magicBuffer = stackalloc byte[HeaderMagicLength];
|
||||
if (reader.Read(magicBuffer) != HeaderMagicLength || !magicBuffer.SequenceEqual(Magic))
|
||||
{
|
||||
throw new InvalidOperationException("Invalid BOM index header magic.");
|
||||
}
|
||||
|
||||
var version = reader.ReadUInt16();
|
||||
if (version != 1)
|
||||
{
|
||||
throw new NotSupportedException($"Unsupported BOM index version '{version}'.");
|
||||
}
|
||||
|
||||
var flags = reader.ReadUInt16();
|
||||
var hasEntrypoints = (flags & 0x1) == 1;
|
||||
|
||||
var digestLength = reader.ReadUInt16();
|
||||
var digestBytes = reader.ReadBytes(digestLength);
|
||||
var imageDigest = Encoding.UTF8.GetString(digestBytes);
|
||||
|
||||
var generatedAtMicros = reader.ReadInt64();
|
||||
var generatedAt = DateTimeOffset.FromUnixTimeMilliseconds(generatedAtMicros / 1000)
|
||||
.AddTicks((generatedAtMicros % 1000) * TimeSpan.TicksPerMillisecond / 1000);
|
||||
|
||||
var layerCount = checked((int)reader.ReadUInt32());
|
||||
var componentCount = checked((int)reader.ReadUInt32());
|
||||
var entrypointCount = checked((int)reader.ReadUInt32());
|
||||
|
||||
// Layer table (we only need to skip entries but validate length)
|
||||
for (var i = 0; i < layerCount; i++)
|
||||
{
|
||||
_ = ReadUtf8String(reader);
|
||||
}
|
||||
|
||||
var componentKeys = new string[componentCount];
|
||||
for (var i = 0; i < componentCount; i++)
|
||||
{
|
||||
componentKeys[i] = ReadUtf8String(reader);
|
||||
}
|
||||
|
||||
for (var i = 0; i < componentCount; i++)
|
||||
{
|
||||
var length = reader.ReadUInt32();
|
||||
if (length > 0)
|
||||
{
|
||||
var payload = reader.ReadBytes(checked((int)length));
|
||||
using var bitmapStream = new MemoryStream(payload, writable: false);
|
||||
_ = RoaringBitmap.Deserialize(bitmapStream);
|
||||
}
|
||||
}
|
||||
|
||||
var entrypointPresence = new bool[componentCount];
|
||||
if (hasEntrypoints && entrypointCount > 0)
|
||||
{
|
||||
// Entrypoint table (skip strings)
|
||||
for (var i = 0; i < entrypointCount; i++)
|
||||
{
|
||||
_ = ReadUtf8String(reader);
|
||||
}
|
||||
|
||||
for (var i = 0; i < componentCount; i++)
|
||||
{
|
||||
var length = reader.ReadUInt32();
|
||||
if (length == 0)
|
||||
{
|
||||
entrypointPresence[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
var payload = reader.ReadBytes(checked((int)length));
|
||||
using var bitmapStream = new MemoryStream(payload, writable: false);
|
||||
var bitmap = RoaringBitmap.Deserialize(bitmapStream);
|
||||
entrypointPresence[i] = bitmap.Any();
|
||||
}
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<BomIndexComponent>(componentCount);
|
||||
for (var i = 0; i < componentCount; i++)
|
||||
{
|
||||
var key = componentKeys[i];
|
||||
builder.Add(new BomIndexComponent(key, entrypointPresence[i]));
|
||||
}
|
||||
|
||||
return new BomIndexDocument(imageDigest, generatedAt, builder.MoveToImmutable());
|
||||
}
|
||||
|
||||
private static string ReadUtf8String(BinaryReader reader)
|
||||
{
|
||||
var length = reader.ReadUInt16();
|
||||
if (length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var bytes = reader.ReadBytes(length);
|
||||
return Encoding.UTF8.GetString(bytes);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex.Ingestion;
|
||||
|
||||
/// <summary>
|
||||
/// Describes a BOM-Index ingestion payload for the scheduler impact index.
|
||||
/// </summary>
|
||||
public sealed record ImpactIndexIngestionRequest
|
||||
{
|
||||
public required string TenantId { get; init; }
|
||||
|
||||
public required string ImageDigest { get; init; }
|
||||
|
||||
public required string Registry { get; init; }
|
||||
|
||||
public required string Repository { get; init; }
|
||||
|
||||
public ImmutableArray<string> Namespaces { get; init; } = ImmutableArray<string>.Empty;
|
||||
|
||||
public ImmutableArray<string> Tags { get; init; } = ImmutableArray<string>.Empty;
|
||||
|
||||
public ImmutableSortedDictionary<string, string> Labels { get; init; } = ImmutableSortedDictionary<string, string>.Empty.WithComparers(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public required Stream BomIndexStream { get; init; }
|
||||
= Stream.Null;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# ImpactIndex Stub Removal Tracker
|
||||
|
||||
- **Created:** 2025-10-20
|
||||
- **Owner:** Scheduler ImpactIndex Guild
|
||||
- **Reference Task:** SCHED-IMPACT-16-300 (fixture-backed stub)
|
||||
|
||||
## Exit Reminder
|
||||
|
||||
Replace `FixtureImpactIndex` with the roaring bitmap-backed implementation once SCHED-IMPACT-16-301/302 are completed, then delete:
|
||||
|
||||
1. Stub classes (`FixtureImpactIndex`, `ImpactIndexStubOptions`, `ImpactIndexServiceCollectionExtensions`).
|
||||
2. Embedded sample fixture wiring in `StellaOps.Scheduler.ImpactIndex.csproj`.
|
||||
3. Temporary unit tests in `StellaOps.Scheduler.ImpactIndex.Tests`.
|
||||
|
||||
Remove this file when the production ImpactIndex replaces the stub.
|
||||
@@ -0,0 +1,481 @@
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using Collections.Special;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scheduler.ImpactIndex.Ingestion;
|
||||
using StellaOps.Scheduler.Models;
|
||||
|
||||
namespace StellaOps.Scheduler.ImpactIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Roaring bitmap-backed implementation of the scheduler impact index.
|
||||
/// </summary>
|
||||
public sealed class RoaringImpactIndex : IImpactIndex
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
|
||||
private readonly Dictionary<string, int> _imageIds = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<int, ImpactImageRecord> _images = new();
|
||||
private readonly Dictionary<string, RoaringBitmap> _containsByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, RoaringBitmap> _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private readonly ILogger<RoaringImpactIndex> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public RoaringImpactIndex(ILogger<RoaringImpactIndex> logger, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task IngestAsync(ImpactIndexIngestionRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(request.BomIndexStream);
|
||||
|
||||
using var buffer = new MemoryStream();
|
||||
await request.BomIndexStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
|
||||
var document = BomIndexReader.Read(buffer);
|
||||
if (!string.Equals(document.ImageDigest, request.ImageDigest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"BOM-Index digest mismatch. Header '{document.ImageDigest}', request '{request.ImageDigest}'.");
|
||||
}
|
||||
|
||||
var tenantId = request.TenantId ?? throw new ArgumentNullException(nameof(request.TenantId));
|
||||
var registry = request.Registry ?? throw new ArgumentNullException(nameof(request.Registry));
|
||||
var repository = request.Repository ?? throw new ArgumentNullException(nameof(request.Repository));
|
||||
|
||||
var namespaces = request.Namespaces.IsDefault ? ImmutableArray<string>.Empty : request.Namespaces;
|
||||
var tags = request.Tags.IsDefault ? ImmutableArray<string>.Empty : request.Tags;
|
||||
var labels = request.Labels.Count == 0
|
||||
? ImmutableSortedDictionary<string, string>.Empty.WithComparers(StringComparer.OrdinalIgnoreCase)
|
||||
: request.Labels;
|
||||
|
||||
var componentKeys = document.Components
|
||||
.Select(component => component.Key)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
var entrypointComponents = document.Components
|
||||
.Where(component => component.UsedByEntrypoint)
|
||||
.Select(component => component.Key)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
var imageId = EnsureImageId(request.ImageDigest);
|
||||
|
||||
if (_images.TryGetValue(imageId, out var existing))
|
||||
{
|
||||
RemoveImageComponents(existing);
|
||||
}
|
||||
|
||||
var metadata = new ImpactImageRecord(
|
||||
imageId,
|
||||
tenantId,
|
||||
request.ImageDigest,
|
||||
registry,
|
||||
repository,
|
||||
namespaces,
|
||||
tags,
|
||||
labels,
|
||||
document.GeneratedAt,
|
||||
componentKeys,
|
||||
entrypointComponents);
|
||||
|
||||
_images[imageId] = metadata;
|
||||
_imageIds[request.ImageDigest] = imageId;
|
||||
|
||||
foreach (var key in componentKeys)
|
||||
{
|
||||
var bitmap = _containsByPurl.GetValueOrDefault(key);
|
||||
_containsByPurl[key] = AddImageToBitmap(bitmap, imageId);
|
||||
}
|
||||
|
||||
foreach (var key in entrypointComponents)
|
||||
{
|
||||
var bitmap = _usedByEntrypointByPurl.GetValueOrDefault(key);
|
||||
_usedByEntrypointByPurl[key] = AddImageToBitmap(bitmap, imageId);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"ImpactIndex ingested BOM-Index for {Digest} ({TenantId}/{Repository}). Components={ComponentCount} EntrypointComponents={EntrypointCount}",
|
||||
request.ImageDigest,
|
||||
tenantId,
|
||||
repository,
|
||||
componentKeys.Length,
|
||||
entrypointComponents.Length);
|
||||
}
|
||||
|
||||
public ValueTask<ImpactSet> ResolveByPurlsAsync(
|
||||
IEnumerable<string> purls,
|
||||
bool usageOnly,
|
||||
Selector selector,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(ResolveByPurlsCore(purls, usageOnly, selector));
|
||||
|
||||
public ValueTask<ImpactSet> ResolveByVulnerabilitiesAsync(
|
||||
IEnumerable<string> vulnerabilityIds,
|
||||
bool usageOnly,
|
||||
Selector selector,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(CreateEmptyImpactSet(selector, usageOnly));
|
||||
|
||||
public ValueTask<ImpactSet> ResolveAllAsync(
|
||||
Selector selector,
|
||||
bool usageOnly,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
|
||||
|
||||
private ImpactSet ResolveByPurlsCore(IEnumerable<string> purls, bool usageOnly, Selector selector)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(purls);
|
||||
ArgumentNullException.ThrowIfNull(selector);
|
||||
|
||||
var normalized = purls
|
||||
.Where(static purl => !string.IsNullOrWhiteSpace(purl))
|
||||
.Select(static purl => purl.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
if (normalized.Length == 0)
|
||||
{
|
||||
return CreateEmptyImpactSet(selector, usageOnly);
|
||||
}
|
||||
|
||||
RoaringBitmap imageIds;
|
||||
lock (_gate)
|
||||
{
|
||||
imageIds = RoaringBitmap.Create(Array.Empty<int>());
|
||||
foreach (var purl in normalized)
|
||||
{
|
||||
if (_containsByPurl.TryGetValue(purl, out var bitmap))
|
||||
{
|
||||
imageIds = imageIds | bitmap;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return BuildImpactSet(imageIds, selector, usageOnly);
|
||||
}
|
||||
|
||||
private ImpactSet ResolveAllCore(Selector selector, bool usageOnly)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(selector);
|
||||
|
||||
RoaringBitmap bitmap;
|
||||
lock (_gate)
|
||||
{
|
||||
var ids = _images.Keys.OrderBy(id => id).ToArray();
|
||||
bitmap = RoaringBitmap.Create(ids);
|
||||
}
|
||||
|
||||
return BuildImpactSet(bitmap, selector, usageOnly);
|
||||
}
|
||||
|
||||
private ImpactSet BuildImpactSet(RoaringBitmap imageIds, Selector selector, bool usageOnly)
|
||||
{
|
||||
var images = new List<ImpactImage>();
|
||||
var latestGeneratedAt = DateTimeOffset.MinValue;
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (var imageId in imageIds)
|
||||
{
|
||||
if (!_images.TryGetValue(imageId, out var metadata))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!ImageMatchesSelector(metadata, selector))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (usageOnly && metadata.EntrypointComponents.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (metadata.GeneratedAt > latestGeneratedAt)
|
||||
{
|
||||
latestGeneratedAt = metadata.GeneratedAt;
|
||||
}
|
||||
|
||||
images.Add(new ImpactImage(
|
||||
metadata.Digest,
|
||||
metadata.Registry,
|
||||
metadata.Repository,
|
||||
metadata.Namespaces,
|
||||
metadata.Tags,
|
||||
metadata.EntrypointComponents.Length > 0,
|
||||
metadata.Labels));
|
||||
}
|
||||
}
|
||||
|
||||
if (images.Count == 0)
|
||||
{
|
||||
return CreateEmptyImpactSet(selector, usageOnly);
|
||||
}
|
||||
|
||||
images.Sort(static (left, right) => string.Compare(left.ImageDigest, right.ImageDigest, StringComparison.Ordinal));
|
||||
|
||||
var generatedAt = latestGeneratedAt == DateTimeOffset.MinValue ? _timeProvider.GetUtcNow() : latestGeneratedAt;
|
||||
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
images.ToImmutableArray(),
|
||||
usageOnly,
|
||||
generatedAt,
|
||||
images.Count,
|
||||
snapshotId: null,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly)
|
||||
{
|
||||
return new ImpactSet(
|
||||
selector,
|
||||
ImmutableArray<ImpactImage>.Empty,
|
||||
usageOnly,
|
||||
_timeProvider.GetUtcNow(),
|
||||
0,
|
||||
snapshotId: null,
|
||||
schemaVersion: SchedulerSchemaVersions.ImpactSet);
|
||||
}
|
||||
|
||||
private static bool ImageMatchesSelector(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
if (selector.TenantId is not null && !string.Equals(selector.TenantId, image.TenantId, StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!MatchesScope(image, selector))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (selector.Digests.Length > 0 && !selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (selector.Repositories.Length > 0)
|
||||
{
|
||||
var repoMatch = selector.Repositories.Any(repo =>
|
||||
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase));
|
||||
if (!repoMatch)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.Namespaces.Length > 0)
|
||||
{
|
||||
if (image.Namespaces.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var namespaceMatch = selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase));
|
||||
if (!namespaceMatch)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.IncludeTags.Length > 0)
|
||||
{
|
||||
if (image.Tags.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var tagMatch = selector.IncludeTags.Any(pattern => image.Tags.Any(tag => MatchesTagPattern(tag, pattern)));
|
||||
if (!tagMatch)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.Labels.Length > 0)
|
||||
{
|
||||
if (image.Labels.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var label in selector.Labels)
|
||||
{
|
||||
if (!image.Labels.TryGetValue(label.Key, out var value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (label.Values.Length > 0 && !label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void RemoveImageComponents(ImpactImageRecord record)
|
||||
{
|
||||
foreach (var key in record.Components)
|
||||
{
|
||||
if (_containsByPurl.TryGetValue(key, out var bitmap))
|
||||
{
|
||||
var updated = RemoveImageFromBitmap(bitmap, record.ImageId);
|
||||
if (updated is null)
|
||||
{
|
||||
_containsByPurl.Remove(key);
|
||||
}
|
||||
else
|
||||
{
|
||||
_containsByPurl[key] = updated;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in record.EntrypointComponents)
|
||||
{
|
||||
if (_usedByEntrypointByPurl.TryGetValue(key, out var bitmap))
|
||||
{
|
||||
var updated = RemoveImageFromBitmap(bitmap, record.ImageId);
|
||||
if (updated is null)
|
||||
{
|
||||
_usedByEntrypointByPurl.Remove(key);
|
||||
}
|
||||
else
|
||||
{
|
||||
_usedByEntrypointByPurl[key] = updated;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static RoaringBitmap AddImageToBitmap(RoaringBitmap? bitmap, int imageId)
|
||||
{
|
||||
if (bitmap is null)
|
||||
{
|
||||
return RoaringBitmap.Create(new[] { imageId });
|
||||
}
|
||||
|
||||
if (bitmap.Any(id => id == imageId))
|
||||
{
|
||||
return bitmap;
|
||||
}
|
||||
|
||||
var merged = bitmap
|
||||
.Concat(new[] { imageId })
|
||||
.Distinct()
|
||||
.OrderBy(id => id)
|
||||
.ToArray();
|
||||
|
||||
return RoaringBitmap.Create(merged);
|
||||
}
|
||||
|
||||
private static RoaringBitmap? RemoveImageFromBitmap(RoaringBitmap bitmap, int imageId)
|
||||
{
|
||||
var remaining = bitmap
|
||||
.Where(id => id != imageId)
|
||||
.OrderBy(id => id)
|
||||
.ToArray();
|
||||
if (remaining.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return RoaringBitmap.Create(remaining);
|
||||
}
|
||||
|
||||
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
|
||||
{
|
||||
return selector.Scope switch
|
||||
{
|
||||
SelectorScope.AllImages => true,
|
||||
SelectorScope.ByDigest => selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase),
|
||||
SelectorScope.ByRepository => selector.Repositories.Any(repo =>
|
||||
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByNamespace => !image.Namespaces.IsDefaultOrEmpty && selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)),
|
||||
SelectorScope.ByLabels => selector.Labels.All(label =>
|
||||
image.Labels.TryGetValue(label.Key, out var value) &&
|
||||
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
|
||||
private static bool MatchesTagPattern(string tag, string pattern)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(pattern))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (pattern == "*")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!pattern.Contains('*') && !pattern.Contains('?'))
|
||||
{
|
||||
return string.Equals(tag, pattern, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
var escaped = Regex.Escape(pattern)
|
||||
.Replace("\\*", ".*")
|
||||
.Replace("\\?", ".");
|
||||
return Regex.IsMatch(tag, $"^{escaped}$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
}
|
||||
|
||||
private int EnsureImageId(string digest)
|
||||
{
|
||||
if (_imageIds.TryGetValue(digest, out var existing))
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
var candidate = ComputeDeterministicId(digest);
|
||||
while (_images.ContainsKey(candidate))
|
||||
{
|
||||
candidate = (candidate + 1) & int.MaxValue;
|
||||
if (candidate == 0)
|
||||
{
|
||||
candidate = 1;
|
||||
}
|
||||
}
|
||||
|
||||
_imageIds[digest] = candidate;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private static int ComputeDeterministicId(string digest)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(digest));
|
||||
for (var offset = 0; offset <= bytes.Length - sizeof(int); offset += sizeof(int))
|
||||
{
|
||||
var value = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset, sizeof(int))) & int.MaxValue;
|
||||
if (value != 0)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return digest.GetHashCode(StringComparison.OrdinalIgnoreCase) & int.MaxValue;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="..\..\samples\scanner\images\**\bom-index.json"
|
||||
Link="Fixtures\%(RecursiveDir)%(Filename)%(Extension)" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="RoaringBitmap" Version="0.0.9" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,10 @@
|
||||
# Scheduler ImpactIndex Task Board (Sprint 16)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCHED-IMPACT-16-300 | DONE (2025-10-20) | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. |
|
||||
| SCHED-IMPACT-16-301 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. |
|
||||
| SCHED-IMPACT-16-302 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. |
|
||||
| SCHED-IMPACT-16-303 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. | Snapshot routine implemented; invalidation tests pass; docs describe recovery. |
|
||||
|
||||
> Removal tracking note: see `src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md` for follow-up actions once the roaring bitmap implementation lands.
|
||||
Reference in New Issue
Block a user