using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using Collections.Special;
using Microsoft.Extensions.Logging;
using StellaOps.Scheduler.ImpactIndex.Ingestion;
using StellaOps.Scheduler.Models;
namespace StellaOps.Scheduler.ImpactIndex;
///
/// Roaring bitmap-backed implementation of the scheduler impact index.
///
public sealed class RoaringImpactIndex : IImpactIndex
{
private readonly object _gate = new();
private readonly Dictionary _imageIds = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary _images = new();
private readonly Dictionary _containsByPurl = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger _logger;
private readonly TimeProvider _timeProvider;
private string? _snapshotId;
public RoaringImpactIndex(ILogger logger, TimeProvider? timeProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public ValueTask RemoveAsync(string imageDigest, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
lock (_gate)
{
if (!_imageIds.TryGetValue(imageDigest, out var imageId))
{
return ValueTask.CompletedTask;
}
if (_images.TryGetValue(imageId, out var record))
{
RemoveImageComponents(record);
_images.Remove(imageId);
}
_imageIds.Remove(imageDigest);
_snapshotId = null;
}
return ValueTask.CompletedTask;
}
public async Task IngestAsync(ImpactIndexIngestionRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(request.BomIndexStream);
using var buffer = new MemoryStream();
await request.BomIndexStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
var document = BomIndexReader.Read(buffer);
if (!string.Equals(document.ImageDigest, request.ImageDigest, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException($"BOM-Index digest mismatch. Header '{document.ImageDigest}', request '{request.ImageDigest}'.");
}
var tenantId = request.TenantId ?? throw new ArgumentNullException(nameof(request.TenantId));
var registry = request.Registry ?? throw new ArgumentNullException(nameof(request.Registry));
var repository = request.Repository ?? throw new ArgumentNullException(nameof(request.Repository));
var namespaces = request.Namespaces.IsDefault ? ImmutableArray.Empty : request.Namespaces;
var tags = request.Tags.IsDefault ? ImmutableArray.Empty : request.Tags;
var labels = request.Labels.Count == 0
? ImmutableSortedDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase)
: request.Labels;
var componentKeys = document.Components
.Select(component => component.Key)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
var entrypointComponents = document.Components
.Where(component => component.UsedByEntrypoint)
.Select(component => component.Key)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
lock (_gate)
{
var imageId = EnsureImageId(request.ImageDigest);
if (_images.TryGetValue(imageId, out var existing))
{
RemoveImageComponents(existing);
}
var metadata = new ImpactImageRecord(
imageId,
tenantId,
request.ImageDigest,
registry,
repository,
namespaces,
tags,
labels,
document.GeneratedAt,
componentKeys,
entrypointComponents);
_images[imageId] = metadata;
_imageIds[request.ImageDigest] = imageId;
foreach (var key in componentKeys)
{
var bitmap = _containsByPurl.GetValueOrDefault(key);
_containsByPurl[key] = AddImageToBitmap(bitmap, imageId);
}
foreach (var key in entrypointComponents)
{
var bitmap = _usedByEntrypointByPurl.GetValueOrDefault(key);
_usedByEntrypointByPurl[key] = AddImageToBitmap(bitmap, imageId);
}
}
_logger.LogInformation(
"ImpactIndex ingested BOM-Index for {Digest} ({TenantId}/{Repository}). Components={ComponentCount} EntrypointComponents={EntrypointCount}",
request.ImageDigest,
tenantId,
repository,
componentKeys.Length,
entrypointComponents.Length);
}
public ValueTask ResolveByPurlsAsync(
IEnumerable purls,
bool usageOnly,
Selector selector,
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(ResolveByPurlsCore(purls, usageOnly, selector));
public ValueTask ResolveByVulnerabilitiesAsync(
IEnumerable vulnerabilityIds,
bool usageOnly,
Selector selector,
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(CreateEmptyImpactSet(selector, usageOnly));
public ValueTask ResolveAllAsync(
Selector selector,
bool usageOnly,
CancellationToken cancellationToken = default)
=> ValueTask.FromResult(ResolveAllCore(selector, usageOnly));
public ValueTask CreateSnapshotAsync(CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
lock (_gate)
{
var orderedImages = _images
.Values
.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase)
.ThenBy(img => img.Repository, StringComparer.OrdinalIgnoreCase)
.ToArray();
var idMap = orderedImages
.Select((image, index) => (image.ImageId, NewId: index))
.ToDictionary(tuple => tuple.ImageId, tuple => tuple.NewId);
var compactedImages = orderedImages
.Select(image => image with { ImageId = idMap[image.ImageId] })
.ToImmutableArray();
ImmutableDictionary> CompactBitmaps(Dictionary source)
{
var builder = ImmutableDictionary.CreateBuilder>(StringComparer.OrdinalIgnoreCase);
foreach (var (key, bitmap) in source)
{
var remapped = bitmap
.Select(id => idMap.TryGetValue(id, out var newId) ? newId : (int?)null)
.Where(id => id.HasValue)
.Select(id => id!.Value)
.Distinct()
.OrderBy(id => id)
.ToImmutableArray();
if (remapped.Length > 0)
{
builder[key] = remapped;
}
}
return builder.ToImmutable();
}
var contains = CompactBitmaps(_containsByPurl);
var usedBy = CompactBitmaps(_usedByEntrypointByPurl);
var generatedAt = orderedImages.Length == 0
? _timeProvider.GetUtcNow()
: orderedImages.Max(img => img.GeneratedAt);
var snapshotId = ComputeSnapshotId(compactedImages, contains, usedBy);
_snapshotId = snapshotId;
var snapshot = new ImpactIndexSnapshot(
generatedAt,
snapshotId,
compactedImages,
contains,
usedBy);
return ValueTask.FromResult(snapshot);
}
}
public ValueTask RestoreSnapshotAsync(ImpactIndexSnapshot snapshot, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(snapshot);
cancellationToken.ThrowIfCancellationRequested();
lock (_gate)
{
_images.Clear();
_imageIds.Clear();
_containsByPurl.Clear();
_usedByEntrypointByPurl.Clear();
foreach (var image in snapshot.Images)
{
_images[image.ImageId] = image;
_imageIds[image.Digest] = image.ImageId;
}
foreach (var kvp in snapshot.ContainsByPurl)
{
_containsByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
}
foreach (var kvp in snapshot.UsedByEntrypointByPurl)
{
_usedByEntrypointByPurl[kvp.Key] = RoaringBitmap.Create(kvp.Value.ToArray());
}
_snapshotId = snapshot.SnapshotId;
}
return ValueTask.CompletedTask;
}
private ImpactSet ResolveByPurlsCore(IEnumerable purls, bool usageOnly, Selector selector)
{
ArgumentNullException.ThrowIfNull(purls);
ArgumentNullException.ThrowIfNull(selector);
var normalized = purls
.Where(static purl => !string.IsNullOrWhiteSpace(purl))
.Select(static purl => purl.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (normalized.Length == 0)
{
return CreateEmptyImpactSet(selector, usageOnly);
}
RoaringBitmap imageIds;
lock (_gate)
{
imageIds = RoaringBitmap.Create(Array.Empty());
foreach (var purl in normalized)
{
if (_containsByPurl.TryGetValue(purl, out var bitmap))
{
imageIds = imageIds | bitmap;
}
}
}
return BuildImpactSet(imageIds, selector, usageOnly);
}
private ImpactSet ResolveAllCore(Selector selector, bool usageOnly)
{
ArgumentNullException.ThrowIfNull(selector);
RoaringBitmap bitmap;
lock (_gate)
{
var ids = _images.Keys.OrderBy(id => id).ToArray();
bitmap = RoaringBitmap.Create(ids);
}
return BuildImpactSet(bitmap, selector, usageOnly);
}
private ImpactSet BuildImpactSet(RoaringBitmap imageIds, Selector selector, bool usageOnly)
{
var images = new List();
var latestGeneratedAt = DateTimeOffset.MinValue;
lock (_gate)
{
foreach (var imageId in imageIds)
{
if (!_images.TryGetValue(imageId, out var metadata))
{
continue;
}
if (!ImageMatchesSelector(metadata, selector))
{
continue;
}
if (usageOnly && metadata.EntrypointComponents.Length == 0)
{
continue;
}
if (metadata.GeneratedAt > latestGeneratedAt)
{
latestGeneratedAt = metadata.GeneratedAt;
}
images.Add(new ImpactImage(
metadata.Digest,
metadata.Registry,
metadata.Repository,
metadata.Namespaces,
metadata.Tags,
metadata.EntrypointComponents.Length > 0,
metadata.Labels));
}
}
if (images.Count == 0)
{
return CreateEmptyImpactSet(selector, usageOnly);
}
images.Sort(static (left, right) => string.Compare(left.ImageDigest, right.ImageDigest, StringComparison.Ordinal));
var generatedAt = latestGeneratedAt == DateTimeOffset.MinValue ? _timeProvider.GetUtcNow() : latestGeneratedAt;
return new ImpactSet(
selector,
images.ToImmutableArray(),
usageOnly,
generatedAt,
images.Count,
snapshotId: _snapshotId,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly)
{
return new ImpactSet(
selector,
ImmutableArray.Empty,
usageOnly,
_timeProvider.GetUtcNow(),
0,
snapshotId: _snapshotId,
schemaVersion: SchedulerSchemaVersions.ImpactSet);
}
private static bool ImageMatchesSelector(ImpactImageRecord image, Selector selector)
{
if (selector.TenantId is not null && !string.Equals(selector.TenantId, image.TenantId, StringComparison.Ordinal))
{
return false;
}
if (!MatchesScope(image, selector))
{
return false;
}
if (selector.Digests.Length > 0 && !selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (selector.Repositories.Length > 0)
{
var repoMatch = selector.Repositories.Any(repo =>
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase));
if (!repoMatch)
{
return false;
}
}
if (selector.Namespaces.Length > 0)
{
if (image.Namespaces.IsDefaultOrEmpty)
{
return false;
}
var namespaceMatch = selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase));
if (!namespaceMatch)
{
return false;
}
}
if (selector.IncludeTags.Length > 0)
{
if (image.Tags.IsDefaultOrEmpty)
{
return false;
}
var tagMatch = selector.IncludeTags.Any(pattern => image.Tags.Any(tag => MatchesTagPattern(tag, pattern)));
if (!tagMatch)
{
return false;
}
}
if (selector.Labels.Length > 0)
{
if (image.Labels.Count == 0)
{
return false;
}
foreach (var label in selector.Labels)
{
if (!image.Labels.TryGetValue(label.Key, out var value))
{
return false;
}
if (label.Values.Length > 0 && !label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))
{
return false;
}
}
}
return true;
}
private void RemoveImageComponents(ImpactImageRecord record)
{
foreach (var key in record.Components)
{
if (_containsByPurl.TryGetValue(key, out var bitmap))
{
var updated = RemoveImageFromBitmap(bitmap, record.ImageId);
if (updated is null)
{
_containsByPurl.Remove(key);
}
else
{
_containsByPurl[key] = updated;
}
}
}
foreach (var key in record.EntrypointComponents)
{
if (_usedByEntrypointByPurl.TryGetValue(key, out var bitmap))
{
var updated = RemoveImageFromBitmap(bitmap, record.ImageId);
if (updated is null)
{
_usedByEntrypointByPurl.Remove(key);
}
else
{
_usedByEntrypointByPurl[key] = updated;
}
}
}
}
private static RoaringBitmap AddImageToBitmap(RoaringBitmap? bitmap, int imageId)
{
if (bitmap is null)
{
return RoaringBitmap.Create(new[] { imageId });
}
if (bitmap.Any(id => id == imageId))
{
return bitmap;
}
var merged = bitmap
.Concat(new[] { imageId })
.Distinct()
.OrderBy(id => id)
.ToArray();
return RoaringBitmap.Create(merged);
}
private static RoaringBitmap? RemoveImageFromBitmap(RoaringBitmap bitmap, int imageId)
{
var remaining = bitmap
.Where(id => id != imageId)
.OrderBy(id => id)
.ToArray();
if (remaining.Length == 0)
{
return null;
}
return RoaringBitmap.Create(remaining);
}
private static bool MatchesScope(ImpactImageRecord image, Selector selector)
{
return selector.Scope switch
{
SelectorScope.AllImages => true,
SelectorScope.ByDigest => selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase),
SelectorScope.ByRepository => selector.Repositories.Any(repo =>
string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) ||
string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)),
SelectorScope.ByNamespace => !image.Namespaces.IsDefaultOrEmpty && selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)),
SelectorScope.ByLabels => selector.Labels.All(label =>
image.Labels.TryGetValue(label.Key, out var value) &&
(label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))),
_ => true,
};
}
private static string ComputeSnapshotId(
ImmutableArray images,
ImmutableDictionary> contains,
ImmutableDictionary> usedBy)
{
var builder = new StringBuilder();
foreach (var image in images.OrderBy(img => img.Digest, StringComparer.OrdinalIgnoreCase))
{
builder.Append(image.Digest).Append('|').Append(image.GeneratedAt.ToUnixTimeSeconds()).Append(';');
}
void AppendMap(ImmutableDictionary> map)
{
foreach (var kvp in map.OrderBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase))
{
builder.Append(kvp.Key).Append('=');
foreach (var id in kvp.Value)
{
builder.Append(id).Append(',');
}
builder.Append('|');
}
}
AppendMap(contains);
AppendMap(usedBy);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
return "snap-" + Convert.ToHexString(hash).ToLowerInvariant();
}
private static bool MatchesTagPattern(string tag, string pattern)
{
if (string.IsNullOrWhiteSpace(pattern))
{
return false;
}
if (pattern == "*")
{
return true;
}
if (!pattern.Contains('*') && !pattern.Contains('?'))
{
return string.Equals(tag, pattern, StringComparison.OrdinalIgnoreCase);
}
var escaped = Regex.Escape(pattern)
.Replace("\\*", ".*")
.Replace("\\?", ".");
return Regex.IsMatch(tag, $"^{escaped}$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
}
private int EnsureImageId(string digest)
{
if (_imageIds.TryGetValue(digest, out var existing))
{
return existing;
}
var candidate = ComputeDeterministicId(digest);
while (_images.ContainsKey(candidate))
{
candidate = (candidate + 1) & int.MaxValue;
if (candidate == 0)
{
candidate = 1;
}
}
_imageIds[digest] = candidate;
return candidate;
}
private static int ComputeDeterministicId(string digest)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(digest));
for (var offset = 0; offset <= bytes.Length - sizeof(int); offset += sizeof(int))
{
var value = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset, sizeof(int))) & int.MaxValue;
if (value != 0)
{
return value;
}
}
return digest.GetHashCode(StringComparison.OrdinalIgnoreCase) & int.MaxValue;
}
}