up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,43 +1,43 @@
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Scanner.Storage.Catalog;
public static class CatalogIdFactory
{
public static string CreateArtifactId(ArtifactDocumentType type, string digest)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
return $"{type.ToString().ToLowerInvariant()}::{NormalizeDigest(digest)}";
}
public static string CreateLinkId(LinkSourceType type, string fromDigest, string artifactId)
{
ArgumentException.ThrowIfNullOrWhiteSpace(fromDigest);
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var input = Encoding.UTF8.GetBytes($"{type}:{NormalizeDigest(fromDigest)}:{artifactId}");
var hash = SHA256.HashData(input);
return Convert.ToHexString(hash).ToLowerInvariant();
}
public static string CreateLifecycleRuleId(string artifactId, string @class)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var normalizedClass = string.IsNullOrWhiteSpace(@class) ? "default" : @class.Trim().ToLowerInvariant();
var payload = Encoding.UTF8.GetBytes($"{artifactId}:{normalizedClass}");
var hash = SHA256.HashData(payload);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static string NormalizeDigest(string digest)
{
if (!digest.Contains(':', StringComparison.Ordinal))
{
return $"sha256:{digest.Trim().ToLowerInvariant()}";
}
var parts = digest.Split(':', 2, StringSplitOptions.TrimEntries);
return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}";
}
}
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Scanner.Storage.Catalog;
public static class CatalogIdFactory
{
public static string CreateArtifactId(ArtifactDocumentType type, string digest)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
return $"{type.ToString().ToLowerInvariant()}::{NormalizeDigest(digest)}";
}
public static string CreateLinkId(LinkSourceType type, string fromDigest, string artifactId)
{
ArgumentException.ThrowIfNullOrWhiteSpace(fromDigest);
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var input = Encoding.UTF8.GetBytes($"{type}:{NormalizeDigest(fromDigest)}:{artifactId}");
var hash = SHA256.HashData(input);
return Convert.ToHexString(hash).ToLowerInvariant();
}
public static string CreateLifecycleRuleId(string artifactId, string @class)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var normalizedClass = string.IsNullOrWhiteSpace(@class) ? "default" : @class.Trim().ToLowerInvariant();
var payload = Encoding.UTF8.GetBytes($"{artifactId}:{normalizedClass}");
var hash = SHA256.HashData(payload);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static string NormalizeDigest(string digest)
{
if (!digest.Contains(':', StringComparison.Ordinal))
{
return $"sha256:{digest.Trim().ToLowerInvariant()}";
}
var parts = digest.Split(':', 2, StringSplitOptions.TrimEntries);
return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}";
}
}

View File

@@ -1,19 +1,19 @@
namespace StellaOps.Scanner.Storage.Catalog;
public sealed class ImageDocument
{
public string ImageDigest { get; set; } = string.Empty;
public string Repository { get; set; } = string.Empty;
public string? Tag { get; set; }
= null;
public string Architecture { get; set; } = string.Empty;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime LastSeenAtUtc { get; set; }
= DateTime.UtcNow;
}
namespace StellaOps.Scanner.Storage.Catalog;
public sealed class ImageDocument
{
public string ImageDigest { get; set; } = string.Empty;
public string Repository { get; set; } = string.Empty;
public string? Tag { get; set; }
= null;
public string Architecture { get; set; } = string.Empty;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime LastSeenAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,37 +1,37 @@
namespace StellaOps.Scanner.Storage.Catalog;
public enum JobState
{
Pending,
Running,
Succeeded,
Failed,
Cancelled,
}
public sealed class JobDocument
{
public string Id { get; set; } = string.Empty;
public string Kind { get; set; } = string.Empty;
public JobState State { get; set; } = JobState.Pending;
public string ArgumentsJson { get; set; }
= "{}";
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime? StartedAtUtc { get; set; }
= null;
public DateTime? CompletedAtUtc { get; set; }
= null;
public DateTime? HeartbeatAtUtc { get; set; }
= null;
public string? Error { get; set; }
= null;
}
namespace StellaOps.Scanner.Storage.Catalog;
public enum JobState
{
Pending,
Running,
Succeeded,
Failed,
Cancelled,
}
public sealed class JobDocument
{
public string Id { get; set; } = string.Empty;
public string Kind { get; set; } = string.Empty;
public JobState State { get; set; } = JobState.Pending;
public string ArgumentsJson { get; set; }
= "{}";
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime? StartedAtUtc { get; set; }
= null;
public DateTime? CompletedAtUtc { get; set; }
= null;
public DateTime? HeartbeatAtUtc { get; set; }
= null;
public string? Error { get; set; }
= null;
}

View File

@@ -1,17 +1,17 @@
namespace StellaOps.Scanner.Storage.Catalog;
public sealed class LayerDocument
{
public string LayerDigest { get; set; } = string.Empty;
public string MediaType { get; set; } = string.Empty;
public long SizeBytes { get; set; }
= 0;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime LastSeenAtUtc { get; set; }
= DateTime.UtcNow;
}
namespace StellaOps.Scanner.Storage.Catalog;
public sealed class LayerDocument
{
public string LayerDigest { get; set; } = string.Empty;
public string MediaType { get; set; } = string.Empty;
public long SizeBytes { get; set; }
= 0;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime LastSeenAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,16 +1,16 @@
namespace StellaOps.Scanner.Storage.Catalog;
public sealed class LifecycleRuleDocument
{
public string Id { get; set; } = string.Empty;
public string ArtifactId { get; set; } = string.Empty;
public string Class { get; set; } = "default";
public DateTime? ExpiresAtUtc { get; set; }
= null;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
}
namespace StellaOps.Scanner.Storage.Catalog;
public sealed class LifecycleRuleDocument
{
public string Id { get; set; } = string.Empty;
public string ArtifactId { get; set; } = string.Empty;
public string Class { get; set; } = "default";
public DateTime? ExpiresAtUtc { get; set; }
= null;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,22 +1,22 @@
namespace StellaOps.Scanner.Storage.Catalog;
public enum LinkSourceType
{
Image,
Layer,
}
public sealed class LinkDocument
{
public string Id { get; set; } = string.Empty;
public LinkSourceType FromType { get; set; }
= LinkSourceType.Image;
public string FromDigest { get; set; } = string.Empty;
public string ArtifactId { get; set; } = string.Empty;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
}
namespace StellaOps.Scanner.Storage.Catalog;
public enum LinkSourceType
{
Image,
Layer,
}
public sealed class LinkDocument
{
public string Id { get; set; } = string.Empty;
public LinkSourceType FromType { get; set; }
= LinkSourceType.Image;
public string FromDigest { get; set; } = string.Empty;
public string ArtifactId { get; set; } = string.Empty;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,53 +1,53 @@
namespace StellaOps.Scanner.Storage.Catalog;
/// <summary>
/// Persistence model for runtime events emitted by the Zastava observer.
/// </summary>
public sealed class RuntimeEventDocument
{
public string? Id { get; set; }
public string EventId { get; set; } = string.Empty;
public string SchemaVersion { get; set; } = string.Empty;
public string Tenant { get; set; } = string.Empty;
public string Node { get; set; } = string.Empty;
public string Kind { get; set; } = string.Empty;
public DateTime When { get; set; }
public DateTime ReceivedAt { get; set; }
public DateTime ExpiresAt { get; set; }
public string? Platform { get; set; }
public string? Namespace { get; set; }
public string? Pod { get; set; }
public string? Container { get; set; }
public string? ContainerId { get; set; }
public string? ImageRef { get; set; }
public string? ImageDigest { get; set; }
public string? Engine { get; set; }
public string? EngineVersion { get; set; }
public string? BaselineDigest { get; set; }
public bool? ImageSigned { get; set; }
public string? SbomReferrer { get; set; }
public string? BuildId { get; set; }
public string PayloadJson { get; set; } = "{}";
}
namespace StellaOps.Scanner.Storage.Catalog;
/// <summary>
/// Persistence model for runtime events emitted by the Zastava observer.
/// </summary>
public sealed class RuntimeEventDocument
{
public string? Id { get; set; }
public string EventId { get; set; } = string.Empty;
public string SchemaVersion { get; set; } = string.Empty;
public string Tenant { get; set; } = string.Empty;
public string Node { get; set; } = string.Empty;
public string Kind { get; set; } = string.Empty;
public DateTime When { get; set; }
public DateTime ReceivedAt { get; set; }
public DateTime ExpiresAt { get; set; }
public string? Platform { get; set; }
public string? Namespace { get; set; }
public string? Pod { get; set; }
public string? Container { get; set; }
public string? ContainerId { get; set; }
public string? ImageRef { get; set; }
public string? ImageDigest { get; set; }
public string? Engine { get; set; }
public string? EngineVersion { get; set; }
public string? BaselineDigest { get; set; }
public bool? ImageSigned { get; set; }
public string? SbomReferrer { get; set; }
public string? BuildId { get; set; }
public string PayloadJson { get; set; } = "{}";
}

View File

@@ -1,173 +1,173 @@
using System;
using System.Net.Http;
using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Infrastructure.Postgres.Migrations;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Postgres;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Storage.Services;
namespace StellaOps.Scanner.Storage.Extensions;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddScannerStorage(this IServiceCollection services, Action<ScannerStorageOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<ScannerStorageOptions>()
.Configure(configure)
.PostConfigure(options => options.EnsureValid())
.ValidateOnStart();
RegisterScannerStorageServices(services);
return services;
}
public static IServiceCollection AddScannerStorage(this IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(configuration);
services.AddOptions<ScannerStorageOptions>()
.Bind(configuration)
.PostConfigure(options => options.EnsureValid())
.ValidateOnStart();
RegisterScannerStorageServices(services);
return services;
}
private static void RegisterScannerStorageServices(IServiceCollection services)
{
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
services.TryAddSingleton<ScannerDataSource>();
services.AddStartupMigrations<ScannerStorageOptions>(
ScannerDataSource.DefaultSchema,
"Scanner.Storage",
typeof(ScannerDataSource).Assembly,
options => options.Postgres.ConnectionString);
services.AddMigrationStatus<ScannerStorageOptions>(
ScannerDataSource.DefaultSchema,
"Scanner.Storage",
typeof(ScannerDataSource).Assembly,
options => options.Postgres.ConnectionString);
services.AddScoped<ArtifactRepository>();
services.AddScoped<ImageRepository>();
services.AddScoped<LayerRepository>();
services.AddScoped<LinkRepository>();
services.AddScoped<JobRepository>();
services.AddScoped<LifecycleRuleRepository>();
services.AddScoped<RuntimeEventRepository>();
services.AddScoped<EntryTraceRepository>();
services.AddScoped<RubyPackageInventoryRepository>();
services.AddScoped<BunPackageInventoryRepository>();
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
services.AddSingleton<IBunPackageInventoryStore, BunPackageInventoryStore>();
services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName)
.ConfigureHttpClient((sp, client) =>
{
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
if (!options.IsRustFsDriver())
{
return;
}
if (!Uri.TryCreate(options.RustFs.BaseUrl, UriKind.Absolute, out var baseUri))
{
throw new InvalidOperationException("RustFS baseUrl must be a valid absolute URI.");
}
client.BaseAddress = baseUri;
client.Timeout = options.RustFs.Timeout;
foreach (var header in options.Headers)
{
client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
}
if (!string.IsNullOrWhiteSpace(options.RustFs.ApiKeyHeader)
&& !string.IsNullOrWhiteSpace(options.RustFs.ApiKey))
{
client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFs.ApiKeyHeader, options.RustFs.ApiKey);
}
})
.ConfigurePrimaryHttpMessageHandler(sp =>
{
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
if (!options.IsRustFsDriver())
{
return new HttpClientHandler();
}
var handler = new HttpClientHandler();
if (options.RustFs.AllowInsecureTls)
{
handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
}
return handler;
});
services.TryAddSingleton(CreateAmazonS3Client);
services.TryAddSingleton<IArtifactObjectStore>(CreateArtifactObjectStore);
services.TryAddSingleton<ArtifactStorageService>();
}
private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
ForcePathStyle = options.ForcePathStyle,
};
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
{
config.ServiceURL = options.ServiceUrl;
}
if (!string.IsNullOrWhiteSpace(options.AccessKeyId) && !string.IsNullOrWhiteSpace(options.SecretAccessKey))
{
AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken)
? new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey)
: new SessionAWSCredentials(options.AccessKeyId, options.SecretAccessKey, options.SessionToken);
return new AmazonS3Client(credentials, config);
}
return new AmazonS3Client(config);
}
private static IArtifactObjectStore CreateArtifactObjectStore(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>();
var objectStore = options.Value.ObjectStore;
if (objectStore.IsRustFsDriver())
{
return new RustFsArtifactObjectStore(
provider.GetRequiredService<IHttpClientFactory>(),
options,
provider.GetRequiredService<ILogger<RustFsArtifactObjectStore>>());
}
return new S3ArtifactObjectStore(
provider.GetRequiredService<IAmazonS3>(),
options,
provider.GetRequiredService<ILogger<S3ArtifactObjectStore>>());
}
}
using System;
using System.Net.Http;
using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Infrastructure.Postgres.Migrations;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Postgres;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Storage.Services;
namespace StellaOps.Scanner.Storage.Extensions;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddScannerStorage(this IServiceCollection services, Action<ScannerStorageOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<ScannerStorageOptions>()
.Configure(configure)
.PostConfigure(options => options.EnsureValid())
.ValidateOnStart();
RegisterScannerStorageServices(services);
return services;
}
public static IServiceCollection AddScannerStorage(this IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(configuration);
services.AddOptions<ScannerStorageOptions>()
.Bind(configuration)
.PostConfigure(options => options.EnsureValid())
.ValidateOnStart();
RegisterScannerStorageServices(services);
return services;
}
private static void RegisterScannerStorageServices(IServiceCollection services)
{
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
services.TryAddSingleton<ScannerDataSource>();
services.AddStartupMigrations<ScannerStorageOptions>(
ScannerDataSource.DefaultSchema,
"Scanner.Storage",
typeof(ScannerDataSource).Assembly,
options => options.Postgres.ConnectionString);
services.AddMigrationStatus<ScannerStorageOptions>(
ScannerDataSource.DefaultSchema,
"Scanner.Storage",
typeof(ScannerDataSource).Assembly,
options => options.Postgres.ConnectionString);
services.AddScoped<ArtifactRepository>();
services.AddScoped<ImageRepository>();
services.AddScoped<LayerRepository>();
services.AddScoped<LinkRepository>();
services.AddScoped<JobRepository>();
services.AddScoped<LifecycleRuleRepository>();
services.AddScoped<RuntimeEventRepository>();
services.AddScoped<EntryTraceRepository>();
services.AddScoped<RubyPackageInventoryRepository>();
services.AddScoped<BunPackageInventoryRepository>();
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
services.AddSingleton<IBunPackageInventoryStore, BunPackageInventoryStore>();
services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName)
.ConfigureHttpClient((sp, client) =>
{
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
if (!options.IsRustFsDriver())
{
return;
}
if (!Uri.TryCreate(options.RustFs.BaseUrl, UriKind.Absolute, out var baseUri))
{
throw new InvalidOperationException("RustFS baseUrl must be a valid absolute URI.");
}
client.BaseAddress = baseUri;
client.Timeout = options.RustFs.Timeout;
foreach (var header in options.Headers)
{
client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
}
if (!string.IsNullOrWhiteSpace(options.RustFs.ApiKeyHeader)
&& !string.IsNullOrWhiteSpace(options.RustFs.ApiKey))
{
client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFs.ApiKeyHeader, options.RustFs.ApiKey);
}
})
.ConfigurePrimaryHttpMessageHandler(sp =>
{
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
if (!options.IsRustFsDriver())
{
return new HttpClientHandler();
}
var handler = new HttpClientHandler();
if (options.RustFs.AllowInsecureTls)
{
handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
}
return handler;
});
services.TryAddSingleton(CreateAmazonS3Client);
services.TryAddSingleton<IArtifactObjectStore>(CreateArtifactObjectStore);
services.TryAddSingleton<ArtifactStorageService>();
}
private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
ForcePathStyle = options.ForcePathStyle,
};
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
{
config.ServiceURL = options.ServiceUrl;
}
if (!string.IsNullOrWhiteSpace(options.AccessKeyId) && !string.IsNullOrWhiteSpace(options.SecretAccessKey))
{
AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken)
? new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey)
: new SessionAWSCredentials(options.AccessKeyId, options.SecretAccessKey, options.SessionToken);
return new AmazonS3Client(credentials, config);
}
return new AmazonS3Client(config);
}
private static IArtifactObjectStore CreateArtifactObjectStore(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>();
var objectStore = options.Value.ObjectStore;
if (objectStore.IsRustFsDriver())
{
return new RustFsArtifactObjectStore(
provider.GetRequiredService<IHttpClientFactory>(),
options,
provider.GetRequiredService<ILogger<RustFsArtifactObjectStore>>());
}
return new S3ArtifactObjectStore(
provider.GetRequiredService<IAmazonS3>(),
options,
provider.GetRequiredService<ILogger<S3ArtifactObjectStore>>());
}
}

View File

@@ -1,12 +1,12 @@
namespace StellaOps.Scanner.Storage.ObjectStore;
public interface IArtifactObjectStore
{
Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken);
Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
}
public sealed record ArtifactObjectDescriptor(string Bucket, string Key, bool Immutable, TimeSpan? RetainFor = null);
namespace StellaOps.Scanner.Storage.ObjectStore;
public interface IArtifactObjectStore
{
Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken);
Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
}
public sealed record ArtifactObjectDescriptor(string Bucket, string Key, bool Immutable, TimeSpan? RetainFor = null);

View File

@@ -1,75 +1,75 @@
using Amazon.S3;
using Amazon.S3.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Scanner.Storage.ObjectStore;
public sealed class S3ArtifactObjectStore : IArtifactObjectStore
{
private readonly IAmazonS3 _s3;
private readonly ObjectStoreOptions _options;
private readonly ILogger<S3ArtifactObjectStore> _logger;
public S3ArtifactObjectStore(IAmazonS3 s3, IOptions<ScannerStorageOptions> options, ILogger<S3ArtifactObjectStore> logger)
{
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.ObjectStore;
}
public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(content);
var request = new PutObjectRequest
{
BucketName = descriptor.Bucket,
Key = descriptor.Key,
InputStream = content,
AutoCloseStream = false,
};
if (descriptor.Immutable && _options.EnableObjectLock)
{
request.ObjectLockMode = ObjectLockMode.Compliance;
if (descriptor.RetainFor is { } retention && retention > TimeSpan.Zero)
{
request.ObjectLockRetainUntilDate = DateTime.UtcNow + retention;
}
else if (_options.ComplianceRetention is { } defaultRetention && defaultRetention > TimeSpan.Zero)
{
request.ObjectLockRetainUntilDate = DateTime.UtcNow + defaultRetention;
}
}
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Uploaded scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
}
public async Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(descriptor);
try
{
var response = await _s3.GetObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
var buffer = new MemoryStream();
await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
return buffer;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Scanner object {Bucket}/{Key} not found", descriptor.Bucket, descriptor.Key);
return null;
}
}
public async Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(descriptor);
await _s3.DeleteObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Deleted scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
}
}
using Amazon.S3;
using Amazon.S3.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Scanner.Storage.ObjectStore;
public sealed class S3ArtifactObjectStore : IArtifactObjectStore
{
private readonly IAmazonS3 _s3;
private readonly ObjectStoreOptions _options;
private readonly ILogger<S3ArtifactObjectStore> _logger;
public S3ArtifactObjectStore(IAmazonS3 s3, IOptions<ScannerStorageOptions> options, ILogger<S3ArtifactObjectStore> logger)
{
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.ObjectStore;
}
public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(content);
var request = new PutObjectRequest
{
BucketName = descriptor.Bucket,
Key = descriptor.Key,
InputStream = content,
AutoCloseStream = false,
};
if (descriptor.Immutable && _options.EnableObjectLock)
{
request.ObjectLockMode = ObjectLockMode.Compliance;
if (descriptor.RetainFor is { } retention && retention > TimeSpan.Zero)
{
request.ObjectLockRetainUntilDate = DateTime.UtcNow + retention;
}
else if (_options.ComplianceRetention is { } defaultRetention && defaultRetention > TimeSpan.Zero)
{
request.ObjectLockRetainUntilDate = DateTime.UtcNow + defaultRetention;
}
}
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Uploaded scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
}
public async Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(descriptor);
try
{
var response = await _s3.GetObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
var buffer = new MemoryStream();
await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
return buffer;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Scanner object {Bucket}/{Key} not found", descriptor.Bucket, descriptor.Key);
return null;
}
}
public async Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(descriptor);
await _s3.DeleteObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Deleted scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
}
}

View File

@@ -1,176 +1,176 @@
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class ArtifactRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.artifacts";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly TimeProvider _timeProvider;
public ArtifactRepository(
ScannerDataSource dataSource,
ILogger<ArtifactRepository> logger,
TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task<ArtifactDocument?> GetAsync(string artifactId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var sql = $"""
SELECT id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
rekor, ttl_class, created_at_utc, updated_at_utc
FROM {Table}
WHERE id = @id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "id", artifactId),
MapArtifact,
cancellationToken);
}
public Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
document.UpdatedAtUtc = now;
var sql = $"""
INSERT INTO {Table} (
id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
rekor, ttl_class, created_at_utc, updated_at_utc
)
VALUES (
@id, @type, @format, @media_type, @bytes_sha256, @size_bytes, @immutable, @ref_count,
@rekor::jsonb, @ttl_class, @created_at_utc, @updated_at_utc
)
ON CONFLICT (id) DO UPDATE SET
type = EXCLUDED.type,
format = EXCLUDED.format,
media_type = EXCLUDED.media_type,
bytes_sha256 = EXCLUDED.bytes_sha256,
size_bytes = EXCLUDED.size_bytes,
immutable = EXCLUDED.immutable,
ref_count = EXCLUDED.ref_count,
rekor = EXCLUDED.rekor,
ttl_class = EXCLUDED.ttl_class,
updated_at_utc = EXCLUDED.updated_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "type", document.Type.ToString());
AddParameter(cmd, "format", document.Format.ToString());
AddParameter(cmd, "media_type", document.MediaType);
AddParameter(cmd, "bytes_sha256", document.BytesSha256);
AddParameter(cmd, "size_bytes", document.SizeBytes);
AddParameter(cmd, "immutable", document.Immutable);
AddParameter(cmd, "ref_count", document.RefCount);
AddJsonbParameter(cmd, "rekor", SerializeRekor(document.Rekor));
AddParameter(cmd, "ttl_class", document.TtlClass);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "updated_at_utc", document.UpdatedAtUtc);
},
cancellationToken);
}
public Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
ArgumentNullException.ThrowIfNull(reference);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var sql = $"""
UPDATE {Table}
SET rekor = @rekor::jsonb,
updated_at_utc = @updated_at_utc
WHERE id = @id
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", artifactId);
AddJsonbParameter(cmd, "rekor", SerializeRekor(reference));
AddParameter(cmd, "updated_at_utc", now);
},
cancellationToken);
}
public async Task<long> IncrementRefCountAsync(string artifactId, long delta, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var sql = $"""
UPDATE {Table}
SET ref_count = ref_count + @delta,
updated_at_utc = NOW() AT TIME ZONE 'UTC'
WHERE id = @id
RETURNING ref_count
""";
var result = await ExecuteScalarAsync<long?>(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", artifactId);
AddParameter(cmd, "delta", delta);
},
cancellationToken).ConfigureAwait(false);
return result ?? 0;
}
private static ArtifactDocument MapArtifact(NpgsqlDataReader reader)
{
var rekorOrdinal = reader.GetOrdinal("rekor");
return new ArtifactDocument
{
Id = reader.GetString(reader.GetOrdinal("id")),
Type = ParseEnum(reader.GetString(reader.GetOrdinal("type")), ArtifactDocumentType.ImageBom),
Format = ParseEnum(reader.GetString(reader.GetOrdinal("format")), ArtifactDocumentFormat.CycloneDxJson),
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
BytesSha256 = reader.GetString(reader.GetOrdinal("bytes_sha256")),
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
Immutable = reader.GetBoolean(reader.GetOrdinal("immutable")),
RefCount = reader.GetInt64(reader.GetOrdinal("ref_count")),
Rekor = reader.IsDBNull(rekorOrdinal)
? null
: JsonSerializer.Deserialize<RekorReference>(reader.GetString(rekorOrdinal), JsonOptions),
TtlClass = reader.GetString(reader.GetOrdinal("ttl_class")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
UpdatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("updated_at_utc")),
};
}
private static TEnum ParseEnum<TEnum>(string value, TEnum fallback) where TEnum : struct
=> Enum.TryParse<TEnum>(value, ignoreCase: true, out var parsed) ? parsed : fallback;
private static string? SerializeRekor(RekorReference? reference)
=> reference is null ? null : JsonSerializer.Serialize(reference, JsonOptions);
}
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class ArtifactRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.artifacts";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly TimeProvider _timeProvider;
public ArtifactRepository(
ScannerDataSource dataSource,
ILogger<ArtifactRepository> logger,
TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task<ArtifactDocument?> GetAsync(string artifactId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var sql = $"""
SELECT id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
rekor, ttl_class, created_at_utc, updated_at_utc
FROM {Table}
WHERE id = @id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "id", artifactId),
MapArtifact,
cancellationToken);
}
public Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
document.UpdatedAtUtc = now;
var sql = $"""
INSERT INTO {Table} (
id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
rekor, ttl_class, created_at_utc, updated_at_utc
)
VALUES (
@id, @type, @format, @media_type, @bytes_sha256, @size_bytes, @immutable, @ref_count,
@rekor::jsonb, @ttl_class, @created_at_utc, @updated_at_utc
)
ON CONFLICT (id) DO UPDATE SET
type = EXCLUDED.type,
format = EXCLUDED.format,
media_type = EXCLUDED.media_type,
bytes_sha256 = EXCLUDED.bytes_sha256,
size_bytes = EXCLUDED.size_bytes,
immutable = EXCLUDED.immutable,
ref_count = EXCLUDED.ref_count,
rekor = EXCLUDED.rekor,
ttl_class = EXCLUDED.ttl_class,
updated_at_utc = EXCLUDED.updated_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "type", document.Type.ToString());
AddParameter(cmd, "format", document.Format.ToString());
AddParameter(cmd, "media_type", document.MediaType);
AddParameter(cmd, "bytes_sha256", document.BytesSha256);
AddParameter(cmd, "size_bytes", document.SizeBytes);
AddParameter(cmd, "immutable", document.Immutable);
AddParameter(cmd, "ref_count", document.RefCount);
AddJsonbParameter(cmd, "rekor", SerializeRekor(document.Rekor));
AddParameter(cmd, "ttl_class", document.TtlClass);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "updated_at_utc", document.UpdatedAtUtc);
},
cancellationToken);
}
public Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
ArgumentNullException.ThrowIfNull(reference);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var sql = $"""
UPDATE {Table}
SET rekor = @rekor::jsonb,
updated_at_utc = @updated_at_utc
WHERE id = @id
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", artifactId);
AddJsonbParameter(cmd, "rekor", SerializeRekor(reference));
AddParameter(cmd, "updated_at_utc", now);
},
cancellationToken);
}
public async Task<long> IncrementRefCountAsync(string artifactId, long delta, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var sql = $"""
UPDATE {Table}
SET ref_count = ref_count + @delta,
updated_at_utc = NOW() AT TIME ZONE 'UTC'
WHERE id = @id
RETURNING ref_count
""";
var result = await ExecuteScalarAsync<long?>(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", artifactId);
AddParameter(cmd, "delta", delta);
},
cancellationToken).ConfigureAwait(false);
return result ?? 0;
}
private static ArtifactDocument MapArtifact(NpgsqlDataReader reader)
{
var rekorOrdinal = reader.GetOrdinal("rekor");
return new ArtifactDocument
{
Id = reader.GetString(reader.GetOrdinal("id")),
Type = ParseEnum(reader.GetString(reader.GetOrdinal("type")), ArtifactDocumentType.ImageBom),
Format = ParseEnum(reader.GetString(reader.GetOrdinal("format")), ArtifactDocumentFormat.CycloneDxJson),
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
BytesSha256 = reader.GetString(reader.GetOrdinal("bytes_sha256")),
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
Immutable = reader.GetBoolean(reader.GetOrdinal("immutable")),
RefCount = reader.GetInt64(reader.GetOrdinal("ref_count")),
Rekor = reader.IsDBNull(rekorOrdinal)
? null
: JsonSerializer.Deserialize<RekorReference>(reader.GetString(rekorOrdinal), JsonOptions),
TtlClass = reader.GetString(reader.GetOrdinal("ttl_class")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
UpdatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("updated_at_utc")),
};
}
private static TEnum ParseEnum<TEnum>(string value, TEnum fallback) where TEnum : struct
=> Enum.TryParse<TEnum>(value, ignoreCase: true, out var parsed) ? parsed : fallback;
private static string? SerializeRekor(RekorReference? reference)
=> reference is null ? null : JsonSerializer.Serialize(reference, JsonOptions);
}

View File

@@ -1,86 +1,86 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class ImageRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.images";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public ImageRepository(ScannerDataSource dataSource, ILogger<ImageRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.LastSeenAtUtc = now;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (
image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
)
VALUES (
@image_digest, @repository, @tag, @architecture, @created_at_utc, @last_seen_at_utc
)
ON CONFLICT (image_digest) DO UPDATE SET
repository = EXCLUDED.repository,
tag = EXCLUDED.tag,
architecture = EXCLUDED.architecture,
last_seen_at_utc = EXCLUDED.last_seen_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "repository", document.Repository);
AddParameter(cmd, "tag", document.Tag);
AddParameter(cmd, "architecture", document.Architecture);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
},
cancellationToken);
}
public Task<ImageDocument?> GetAsync(string imageDigest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
var sql = $"""
SELECT image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
FROM {Table}
WHERE image_digest = @image_digest
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "image_digest", imageDigest),
MapImage,
cancellationToken);
}
private static ImageDocument MapImage(NpgsqlDataReader reader) => new()
{
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
Repository = reader.GetString(reader.GetOrdinal("repository")),
Tag = GetNullableString(reader, reader.GetOrdinal("tag")),
Architecture = reader.GetString(reader.GetOrdinal("architecture")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
};
}
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class ImageRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.images";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public ImageRepository(ScannerDataSource dataSource, ILogger<ImageRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.LastSeenAtUtc = now;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (
image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
)
VALUES (
@image_digest, @repository, @tag, @architecture, @created_at_utc, @last_seen_at_utc
)
ON CONFLICT (image_digest) DO UPDATE SET
repository = EXCLUDED.repository,
tag = EXCLUDED.tag,
architecture = EXCLUDED.architecture,
last_seen_at_utc = EXCLUDED.last_seen_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "repository", document.Repository);
AddParameter(cmd, "tag", document.Tag);
AddParameter(cmd, "architecture", document.Architecture);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
},
cancellationToken);
}
public Task<ImageDocument?> GetAsync(string imageDigest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
var sql = $"""
SELECT image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
FROM {Table}
WHERE image_digest = @image_digest
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "image_digest", imageDigest),
MapImage,
cancellationToken);
}
private static ImageDocument MapImage(NpgsqlDataReader reader) => new()
{
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
Repository = reader.GetString(reader.GetOrdinal("repository")),
Tag = GetNullableString(reader, reader.GetOrdinal("tag")),
Architecture = reader.GetString(reader.GetOrdinal("architecture")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
};
}

View File

@@ -1,151 +1,151 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class JobRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.jobs";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public JobRepository(ScannerDataSource dataSource, ILogger<JobRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<JobDocument> InsertAsync(JobDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = now;
document.HeartbeatAtUtc = now;
var sql = $"""
INSERT INTO {Table} (
id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
)
VALUES (
@id, @kind, @state::job_state, @args::jsonb, @created_at_utc, @started_at_utc, @completed_at_utc, @heartbeat_at_utc, @error
)
RETURNING id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
""";
var inserted = await QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "kind", document.Kind);
AddParameter(cmd, "state", document.State.ToString());
AddJsonbParameter(cmd, "args", document.ArgumentsJson);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "started_at_utc", document.StartedAtUtc);
AddParameter(cmd, "completed_at_utc", document.CompletedAtUtc);
AddParameter(cmd, "heartbeat_at_utc", document.HeartbeatAtUtc);
AddParameter(cmd, "error", document.Error);
},
MapJob,
cancellationToken).ConfigureAwait(false);
return inserted ?? document;
}
public async Task<bool> TryTransitionAsync(string jobId, JobState expected, JobState next, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var sql = $"""
UPDATE {Table}
SET state = @next::job_state,
heartbeat_at_utc = @heartbeat,
started_at_utc = CASE
WHEN @next = 'Running' THEN COALESCE(started_at_utc, @heartbeat)
ELSE started_at_utc END,
completed_at_utc = CASE
WHEN @next IN ('Succeeded','Failed','Cancelled') THEN @heartbeat
ELSE completed_at_utc END
WHERE id = @id AND state = @expected::job_state
""";
var affected = await ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", jobId);
AddParameter(cmd, "expected", expected.ToString());
AddParameter(cmd, "next", next.ToString());
AddParameter(cmd, "heartbeat", now);
},
cancellationToken).ConfigureAwait(false);
return affected == 1;
}
public Task<JobDocument?> GetAsync(string jobId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
var sql = $"""
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
FROM {Table}
WHERE id = @id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "id", jobId),
MapJob,
cancellationToken);
}
public Task<List<JobDocument>> ListStaleAsync(TimeSpan heartbeatThreshold, CancellationToken cancellationToken)
{
if (heartbeatThreshold <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(heartbeatThreshold));
}
var cutoff = _timeProvider.GetUtcNow().UtcDateTime - heartbeatThreshold;
var sql = $"""
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
FROM {Table}
WHERE state = 'Running'::job_state
AND heartbeat_at_utc < @cutoff
ORDER BY heartbeat_at_utc
""";
return QueryAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "cutoff", cutoff),
MapJob,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static JobDocument MapJob(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
State = Enum.TryParse<JobState>(reader.GetString(reader.GetOrdinal("state")), true, out var state)
? state
: JobState.Pending,
ArgumentsJson = reader.GetString(reader.GetOrdinal("args")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
StartedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("started_at_utc"))?.UtcDateTime,
CompletedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("completed_at_utc"))?.UtcDateTime,
HeartbeatAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("heartbeat_at_utc"))?.UtcDateTime,
Error = GetNullableString(reader, reader.GetOrdinal("error"))
};
}
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class JobRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.jobs";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public JobRepository(ScannerDataSource dataSource, ILogger<JobRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<JobDocument> InsertAsync(JobDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = now;
document.HeartbeatAtUtc = now;
var sql = $"""
INSERT INTO {Table} (
id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
)
VALUES (
@id, @kind, @state::job_state, @args::jsonb, @created_at_utc, @started_at_utc, @completed_at_utc, @heartbeat_at_utc, @error
)
RETURNING id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
""";
var inserted = await QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "kind", document.Kind);
AddParameter(cmd, "state", document.State.ToString());
AddJsonbParameter(cmd, "args", document.ArgumentsJson);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "started_at_utc", document.StartedAtUtc);
AddParameter(cmd, "completed_at_utc", document.CompletedAtUtc);
AddParameter(cmd, "heartbeat_at_utc", document.HeartbeatAtUtc);
AddParameter(cmd, "error", document.Error);
},
MapJob,
cancellationToken).ConfigureAwait(false);
return inserted ?? document;
}
public async Task<bool> TryTransitionAsync(string jobId, JobState expected, JobState next, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var sql = $"""
UPDATE {Table}
SET state = @next::job_state,
heartbeat_at_utc = @heartbeat,
started_at_utc = CASE
WHEN @next = 'Running' THEN COALESCE(started_at_utc, @heartbeat)
ELSE started_at_utc END,
completed_at_utc = CASE
WHEN @next IN ('Succeeded','Failed','Cancelled') THEN @heartbeat
ELSE completed_at_utc END
WHERE id = @id AND state = @expected::job_state
""";
var affected = await ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", jobId);
AddParameter(cmd, "expected", expected.ToString());
AddParameter(cmd, "next", next.ToString());
AddParameter(cmd, "heartbeat", now);
},
cancellationToken).ConfigureAwait(false);
return affected == 1;
}
public Task<JobDocument?> GetAsync(string jobId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
var sql = $"""
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
FROM {Table}
WHERE id = @id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "id", jobId),
MapJob,
cancellationToken);
}
public Task<List<JobDocument>> ListStaleAsync(TimeSpan heartbeatThreshold, CancellationToken cancellationToken)
{
if (heartbeatThreshold <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(heartbeatThreshold));
}
var cutoff = _timeProvider.GetUtcNow().UtcDateTime - heartbeatThreshold;
var sql = $"""
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
FROM {Table}
WHERE state = 'Running'::job_state
AND heartbeat_at_utc < @cutoff
ORDER BY heartbeat_at_utc
""";
return QueryAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "cutoff", cutoff),
MapJob,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static JobDocument MapJob(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
State = Enum.TryParse<JobState>(reader.GetString(reader.GetOrdinal("state")), true, out var state)
? state
: JobState.Pending,
ArgumentsJson = reader.GetString(reader.GetOrdinal("args")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
StartedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("started_at_utc"))?.UtcDateTime,
CompletedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("completed_at_utc"))?.UtcDateTime,
HeartbeatAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("heartbeat_at_utc"))?.UtcDateTime,
Error = GetNullableString(reader, reader.GetOrdinal("error"))
};
}

View File

@@ -1,79 +1,79 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LayerRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.layers";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public LayerRepository(ScannerDataSource dataSource, ILogger<LayerRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.LastSeenAtUtc = now;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc)
VALUES (@layer_digest, @media_type, @size_bytes, @created_at_utc, @last_seen_at_utc)
ON CONFLICT (layer_digest) DO UPDATE SET
media_type = EXCLUDED.media_type,
size_bytes = EXCLUDED.size_bytes,
last_seen_at_utc = EXCLUDED.last_seen_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "layer_digest", document.LayerDigest);
AddParameter(cmd, "media_type", document.MediaType);
AddParameter(cmd, "size_bytes", document.SizeBytes);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
},
cancellationToken);
}
public Task<LayerDocument?> GetAsync(string layerDigest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
var sql = $"""
SELECT layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc
FROM {Table}
WHERE layer_digest = @layer_digest
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "layer_digest", layerDigest),
MapLayer,
cancellationToken);
}
private static LayerDocument MapLayer(NpgsqlDataReader reader) => new()
{
LayerDigest = reader.GetString(reader.GetOrdinal("layer_digest")),
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
};
}
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LayerRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.layers";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public LayerRepository(ScannerDataSource dataSource, ILogger<LayerRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.LastSeenAtUtc = now;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc)
VALUES (@layer_digest, @media_type, @size_bytes, @created_at_utc, @last_seen_at_utc)
ON CONFLICT (layer_digest) DO UPDATE SET
media_type = EXCLUDED.media_type,
size_bytes = EXCLUDED.size_bytes,
last_seen_at_utc = EXCLUDED.last_seen_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "layer_digest", document.LayerDigest);
AddParameter(cmd, "media_type", document.MediaType);
AddParameter(cmd, "size_bytes", document.SizeBytes);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
},
cancellationToken);
}
public Task<LayerDocument?> GetAsync(string layerDigest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
var sql = $"""
SELECT layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc
FROM {Table}
WHERE layer_digest = @layer_digest
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "layer_digest", layerDigest),
MapLayer,
cancellationToken);
}
private static LayerDocument MapLayer(NpgsqlDataReader reader) => new()
{
LayerDigest = reader.GetString(reader.GetOrdinal("layer_digest")),
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
};
}

View File

@@ -1,77 +1,77 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LifecycleRuleRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.lifecycle_rules";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public LifecycleRuleRepository(ScannerDataSource dataSource, ILogger<LifecycleRuleRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (id, artifact_id, class, expires_at_utc, created_at_utc)
VALUES (@id, @artifact_id, @class, @expires_at_utc, @created_at_utc)
ON CONFLICT (id) DO UPDATE SET
artifact_id = EXCLUDED.artifact_id,
class = EXCLUDED.class,
expires_at_utc = EXCLUDED.expires_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "artifact_id", document.ArtifactId);
AddParameter(cmd, "class", document.Class);
AddParameter(cmd, "expires_at_utc", document.ExpiresAtUtc);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
},
cancellationToken);
}
public Task<List<LifecycleRuleDocument>> ListExpiredAsync(DateTime utcNow, CancellationToken cancellationToken)
{
var sql = $"""
SELECT id, artifact_id, class, expires_at_utc, created_at_utc
FROM {Table}
WHERE expires_at_utc IS NOT NULL AND expires_at_utc < @now
ORDER BY expires_at_utc
""";
return QueryAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "now", utcNow),
MapRule,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static LifecycleRuleDocument MapRule(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
Class = reader.GetString(reader.GetOrdinal("class")),
ExpiresAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("expires_at_utc"))?.UtcDateTime,
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
};
}
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LifecycleRuleRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.lifecycle_rules";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public LifecycleRuleRepository(ScannerDataSource dataSource, ILogger<LifecycleRuleRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (id, artifact_id, class, expires_at_utc, created_at_utc)
VALUES (@id, @artifact_id, @class, @expires_at_utc, @created_at_utc)
ON CONFLICT (id) DO UPDATE SET
artifact_id = EXCLUDED.artifact_id,
class = EXCLUDED.class,
expires_at_utc = EXCLUDED.expires_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "artifact_id", document.ArtifactId);
AddParameter(cmd, "class", document.Class);
AddParameter(cmd, "expires_at_utc", document.ExpiresAtUtc);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
},
cancellationToken);
}
public Task<List<LifecycleRuleDocument>> ListExpiredAsync(DateTime utcNow, CancellationToken cancellationToken)
{
var sql = $"""
SELECT id, artifact_id, class, expires_at_utc, created_at_utc
FROM {Table}
WHERE expires_at_utc IS NOT NULL AND expires_at_utc < @now
ORDER BY expires_at_utc
""";
return QueryAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "now", utcNow),
MapRule,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static LifecycleRuleDocument MapRule(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
Class = reader.GetString(reader.GetOrdinal("class")),
ExpiresAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("expires_at_utc"))?.UtcDateTime,
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
};
}

View File

@@ -1,80 +1,80 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LinkRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.links";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
public LinkRepository(ScannerDataSource dataSource, ILogger<LinkRepository> logger)
: base(dataSource, logger)
{
}
public Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var sql = $"""
INSERT INTO {Table} (id, from_type, from_digest, artifact_id, created_at_utc)
VALUES (@id, @from_type, @from_digest, @artifact_id, @created_at_utc)
ON CONFLICT (id) DO UPDATE SET
from_type = EXCLUDED.from_type,
from_digest = EXCLUDED.from_digest,
artifact_id = EXCLUDED.artifact_id
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "from_type", document.FromType.ToString());
AddParameter(cmd, "from_digest", document.FromDigest);
AddParameter(cmd, "artifact_id", document.ArtifactId);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
},
cancellationToken);
}
public Task<List<LinkDocument>> ListBySourceAsync(LinkSourceType type, string digest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
var sql = $"""
SELECT id, from_type, from_digest, artifact_id, created_at_utc
FROM {Table}
WHERE from_type = @from_type AND from_digest = @from_digest
ORDER BY created_at_utc DESC, id
""";
return QueryAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "from_type", type.ToString());
AddParameter(cmd, "from_digest", digest);
},
MapLink,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static LinkDocument MapLink(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
FromType = Enum.TryParse<LinkSourceType>(reader.GetString(reader.GetOrdinal("from_type")), true, out var parsed)
? parsed
: LinkSourceType.Image,
FromDigest = reader.GetString(reader.GetOrdinal("from_digest")),
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
};
}
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LinkRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.links";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
public LinkRepository(ScannerDataSource dataSource, ILogger<LinkRepository> logger)
: base(dataSource, logger)
{
}
public Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var sql = $"""
INSERT INTO {Table} (id, from_type, from_digest, artifact_id, created_at_utc)
VALUES (@id, @from_type, @from_digest, @artifact_id, @created_at_utc)
ON CONFLICT (id) DO UPDATE SET
from_type = EXCLUDED.from_type,
from_digest = EXCLUDED.from_digest,
artifact_id = EXCLUDED.artifact_id
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "from_type", document.FromType.ToString());
AddParameter(cmd, "from_digest", document.FromDigest);
AddParameter(cmd, "artifact_id", document.ArtifactId);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
},
cancellationToken);
}
public Task<List<LinkDocument>> ListBySourceAsync(LinkSourceType type, string digest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
var sql = $"""
SELECT id, from_type, from_digest, artifact_id, created_at_utc
FROM {Table}
WHERE from_type = @from_type AND from_digest = @from_digest
ORDER BY created_at_utc DESC, id
""";
return QueryAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "from_type", type.ToString());
AddParameter(cmd, "from_digest", digest);
},
MapLink,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static LinkDocument MapLink(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
FromType = Enum.TryParse<LinkSourceType>(reader.GetString(reader.GetOrdinal("from_type")), true, out var parsed)
? parsed
: LinkSourceType.Image,
FromDigest = reader.GetString(reader.GetOrdinal("from_digest")),
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
};
}

View File

@@ -1,247 +1,247 @@
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
/// <summary>
/// Repository responsible for persisting runtime events.
/// </summary>
public sealed class RuntimeEventRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.runtime_events";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
public RuntimeEventRepository(ScannerDataSource dataSource, ILogger<RuntimeEventRepository> logger)
: base(dataSource, logger)
{
}
public async Task<RuntimeEventInsertResult> InsertAsync(
IReadOnlyCollection<RuntimeEventDocument> documents,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(documents);
if (documents.Count == 0)
{
return RuntimeEventInsertResult.Empty;
}
var sql = $"""
INSERT INTO {Table} (
id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
platform, namespace, pod, container, container_id, image_ref, image_digest,
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
)
VALUES (
@id, @event_id, @schema_version, @tenant, @node, @kind, @when, @received_at, @expires_at,
@platform, @namespace, @pod, @container, @container_id, @image_ref, @image_digest,
@engine, @engine_version, @baseline_digest, @image_signed, @sbom_referrer, @build_id, @payload::jsonb
)
ON CONFLICT (event_id) DO NOTHING
""";
var inserted = 0;
var duplicates = 0;
foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
var id = string.IsNullOrWhiteSpace(document.Id) ? Guid.NewGuid().ToString("N") : document.Id;
var rows = await ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", id);
AddParameter(cmd, "event_id", document.EventId);
AddParameter(cmd, "schema_version", document.SchemaVersion);
AddParameter(cmd, "tenant", document.Tenant);
AddParameter(cmd, "node", document.Node);
AddParameter(cmd, "kind", document.Kind);
AddParameter(cmd, "when", document.When);
AddParameter(cmd, "received_at", document.ReceivedAt);
AddParameter(cmd, "expires_at", document.ExpiresAt);
AddParameter(cmd, "platform", document.Platform);
AddParameter(cmd, "namespace", document.Namespace);
AddParameter(cmd, "pod", document.Pod);
AddParameter(cmd, "container", document.Container);
AddParameter(cmd, "container_id", document.ContainerId);
AddParameter(cmd, "image_ref", document.ImageRef);
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "engine", document.Engine);
AddParameter(cmd, "engine_version", document.EngineVersion);
AddParameter(cmd, "baseline_digest", document.BaselineDigest);
AddParameter(cmd, "image_signed", document.ImageSigned);
AddParameter(cmd, "sbom_referrer", document.SbomReferrer);
AddParameter(cmd, "build_id", document.BuildId);
AddJsonbParameter(cmd, "payload", document.PayloadJson);
},
cancellationToken).ConfigureAwait(false);
if (rows > 0)
{
inserted++;
}
else
{
duplicates++;
}
}
return new RuntimeEventInsertResult(inserted, duplicates);
}
public async Task<IReadOnlyDictionary<string, RuntimeBuildIdObservation>> GetRecentBuildIdsAsync(
IReadOnlyCollection<string> imageDigests,
int maxPerImage,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(imageDigests);
if (imageDigests.Count == 0 || maxPerImage <= 0)
{
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
}
var normalized = imageDigests
.Where(digest => !string.IsNullOrWhiteSpace(digest))
.Select(digest => digest.Trim().ToLowerInvariant())
.Distinct(StringComparer.Ordinal)
.ToArray();
if (normalized.Length == 0)
{
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
}
var sql = $"""
SELECT image_digest, build_id, "when"
FROM {Table}
WHERE image_digest = ANY(@digests)
AND build_id IS NOT NULL
AND build_id <> ''
ORDER BY image_digest, "when" DESC
""";
var rows = await QueryAsync(
Tenant,
sql,
cmd => AddTextArrayParameter(cmd, "digests", normalized),
reader => new
{
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
BuildId = reader.GetString(reader.GetOrdinal("build_id")),
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when"))
},
cancellationToken).ConfigureAwait(false);
var result = new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
foreach (var group in rows.GroupBy(r => r.ImageDigest, StringComparer.OrdinalIgnoreCase))
{
var buildIds = group
.Select(r => r.BuildId)
.Where(id => !string.IsNullOrWhiteSpace(id))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(maxPerImage)
.Select(id => id.Trim().ToLowerInvariant())
.ToArray();
if (buildIds.Length == 0)
{
continue;
}
var observedAt = group.Select(r => r.When).FirstOrDefault();
result[group.Key.ToLowerInvariant()] = new RuntimeBuildIdObservation(group.Key, buildIds, observedAt);
}
return result;
}
public Task<int> CountAsync(CancellationToken cancellationToken)
{
var sql = $"""
SELECT COUNT(*) FROM {Table}
""";
return ExecuteScalarAsync<int>(
Tenant,
sql,
null,
cancellationToken);
}
public Task TruncateAsync(CancellationToken cancellationToken)
{
var sql = $"""
TRUNCATE TABLE {Table} RESTART IDENTITY CASCADE
""";
return ExecuteAsync(Tenant, sql, null, cancellationToken);
}
public Task<IReadOnlyList<RuntimeEventDocument>> ListAsync(CancellationToken cancellationToken)
{
var sql = $"""
SELECT id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
platform, namespace, pod, container, container_id, image_ref, image_digest,
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
FROM {Table}
ORDER BY received_at
""";
return QueryAsync(
Tenant,
sql,
null,
MapRuntimeEvent,
cancellationToken);
}
private static RuntimeEventDocument MapRuntimeEvent(NpgsqlDataReader reader)
{
var payloadOrdinal = reader.GetOrdinal("payload");
return new RuntimeEventDocument
{
Id = reader.GetString(reader.GetOrdinal("id")),
EventId = reader.GetString(reader.GetOrdinal("event_id")),
SchemaVersion = reader.GetString(reader.GetOrdinal("schema_version")),
Tenant = reader.GetString(reader.GetOrdinal("tenant")),
Node = reader.GetString(reader.GetOrdinal("node")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when")),
ReceivedAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("received_at")),
ExpiresAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("expires_at")),
Platform = GetNullableString(reader, reader.GetOrdinal("platform")),
Namespace = GetNullableString(reader, reader.GetOrdinal("namespace")),
Pod = GetNullableString(reader, reader.GetOrdinal("pod")),
Container = GetNullableString(reader, reader.GetOrdinal("container")),
ContainerId = GetNullableString(reader, reader.GetOrdinal("container_id")),
ImageRef = GetNullableString(reader, reader.GetOrdinal("image_ref")),
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
Engine = GetNullableString(reader, reader.GetOrdinal("engine")),
EngineVersion = GetNullableString(reader, reader.GetOrdinal("engine_version")),
BaselineDigest = GetNullableString(reader, reader.GetOrdinal("baseline_digest")),
ImageSigned = GetNullableBoolean(reader, reader.GetOrdinal("image_signed")),
SbomReferrer = GetNullableString(reader, reader.GetOrdinal("sbom_referrer")),
BuildId = GetNullableString(reader, reader.GetOrdinal("build_id")),
PayloadJson = reader.IsDBNull(payloadOrdinal) ? "{}" : reader.GetString(payloadOrdinal)
};
}
}
public readonly record struct RuntimeEventInsertResult(int InsertedCount, int DuplicateCount)
{
public static RuntimeEventInsertResult Empty => new(0, 0);
}
public sealed record RuntimeBuildIdObservation(
string ImageDigest,
IReadOnlyList<string> BuildIds,
DateTime ObservedAtUtc);
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
/// <summary>
/// Repository responsible for persisting runtime events.
/// </summary>
public sealed class RuntimeEventRepository : RepositoryBase<ScannerDataSource>
{
private const string Tenant = "";
private string Table => $"{SchemaName}.runtime_events";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
public RuntimeEventRepository(ScannerDataSource dataSource, ILogger<RuntimeEventRepository> logger)
: base(dataSource, logger)
{
}
public async Task<RuntimeEventInsertResult> InsertAsync(
IReadOnlyCollection<RuntimeEventDocument> documents,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(documents);
if (documents.Count == 0)
{
return RuntimeEventInsertResult.Empty;
}
var sql = $"""
INSERT INTO {Table} (
id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
platform, namespace, pod, container, container_id, image_ref, image_digest,
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
)
VALUES (
@id, @event_id, @schema_version, @tenant, @node, @kind, @when, @received_at, @expires_at,
@platform, @namespace, @pod, @container, @container_id, @image_ref, @image_digest,
@engine, @engine_version, @baseline_digest, @image_signed, @sbom_referrer, @build_id, @payload::jsonb
)
ON CONFLICT (event_id) DO NOTHING
""";
var inserted = 0;
var duplicates = 0;
foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
var id = string.IsNullOrWhiteSpace(document.Id) ? Guid.NewGuid().ToString("N") : document.Id;
var rows = await ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", id);
AddParameter(cmd, "event_id", document.EventId);
AddParameter(cmd, "schema_version", document.SchemaVersion);
AddParameter(cmd, "tenant", document.Tenant);
AddParameter(cmd, "node", document.Node);
AddParameter(cmd, "kind", document.Kind);
AddParameter(cmd, "when", document.When);
AddParameter(cmd, "received_at", document.ReceivedAt);
AddParameter(cmd, "expires_at", document.ExpiresAt);
AddParameter(cmd, "platform", document.Platform);
AddParameter(cmd, "namespace", document.Namespace);
AddParameter(cmd, "pod", document.Pod);
AddParameter(cmd, "container", document.Container);
AddParameter(cmd, "container_id", document.ContainerId);
AddParameter(cmd, "image_ref", document.ImageRef);
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "engine", document.Engine);
AddParameter(cmd, "engine_version", document.EngineVersion);
AddParameter(cmd, "baseline_digest", document.BaselineDigest);
AddParameter(cmd, "image_signed", document.ImageSigned);
AddParameter(cmd, "sbom_referrer", document.SbomReferrer);
AddParameter(cmd, "build_id", document.BuildId);
AddJsonbParameter(cmd, "payload", document.PayloadJson);
},
cancellationToken).ConfigureAwait(false);
if (rows > 0)
{
inserted++;
}
else
{
duplicates++;
}
}
return new RuntimeEventInsertResult(inserted, duplicates);
}
public async Task<IReadOnlyDictionary<string, RuntimeBuildIdObservation>> GetRecentBuildIdsAsync(
IReadOnlyCollection<string> imageDigests,
int maxPerImage,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(imageDigests);
if (imageDigests.Count == 0 || maxPerImage <= 0)
{
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
}
var normalized = imageDigests
.Where(digest => !string.IsNullOrWhiteSpace(digest))
.Select(digest => digest.Trim().ToLowerInvariant())
.Distinct(StringComparer.Ordinal)
.ToArray();
if (normalized.Length == 0)
{
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
}
var sql = $"""
SELECT image_digest, build_id, "when"
FROM {Table}
WHERE image_digest = ANY(@digests)
AND build_id IS NOT NULL
AND build_id <> ''
ORDER BY image_digest, "when" DESC
""";
var rows = await QueryAsync(
Tenant,
sql,
cmd => AddTextArrayParameter(cmd, "digests", normalized),
reader => new
{
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
BuildId = reader.GetString(reader.GetOrdinal("build_id")),
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when"))
},
cancellationToken).ConfigureAwait(false);
var result = new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
foreach (var group in rows.GroupBy(r => r.ImageDigest, StringComparer.OrdinalIgnoreCase))
{
var buildIds = group
.Select(r => r.BuildId)
.Where(id => !string.IsNullOrWhiteSpace(id))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(maxPerImage)
.Select(id => id.Trim().ToLowerInvariant())
.ToArray();
if (buildIds.Length == 0)
{
continue;
}
var observedAt = group.Select(r => r.When).FirstOrDefault();
result[group.Key.ToLowerInvariant()] = new RuntimeBuildIdObservation(group.Key, buildIds, observedAt);
}
return result;
}
public Task<int> CountAsync(CancellationToken cancellationToken)
{
var sql = $"""
SELECT COUNT(*) FROM {Table}
""";
return ExecuteScalarAsync<int>(
Tenant,
sql,
null,
cancellationToken);
}
public Task TruncateAsync(CancellationToken cancellationToken)
{
var sql = $"""
TRUNCATE TABLE {Table} RESTART IDENTITY CASCADE
""";
return ExecuteAsync(Tenant, sql, null, cancellationToken);
}
public Task<IReadOnlyList<RuntimeEventDocument>> ListAsync(CancellationToken cancellationToken)
{
var sql = $"""
SELECT id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
platform, namespace, pod, container, container_id, image_ref, image_digest,
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
FROM {Table}
ORDER BY received_at
""";
return QueryAsync(
Tenant,
sql,
null,
MapRuntimeEvent,
cancellationToken);
}
private static RuntimeEventDocument MapRuntimeEvent(NpgsqlDataReader reader)
{
var payloadOrdinal = reader.GetOrdinal("payload");
return new RuntimeEventDocument
{
Id = reader.GetString(reader.GetOrdinal("id")),
EventId = reader.GetString(reader.GetOrdinal("event_id")),
SchemaVersion = reader.GetString(reader.GetOrdinal("schema_version")),
Tenant = reader.GetString(reader.GetOrdinal("tenant")),
Node = reader.GetString(reader.GetOrdinal("node")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when")),
ReceivedAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("received_at")),
ExpiresAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("expires_at")),
Platform = GetNullableString(reader, reader.GetOrdinal("platform")),
Namespace = GetNullableString(reader, reader.GetOrdinal("namespace")),
Pod = GetNullableString(reader, reader.GetOrdinal("pod")),
Container = GetNullableString(reader, reader.GetOrdinal("container")),
ContainerId = GetNullableString(reader, reader.GetOrdinal("container_id")),
ImageRef = GetNullableString(reader, reader.GetOrdinal("image_ref")),
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
Engine = GetNullableString(reader, reader.GetOrdinal("engine")),
EngineVersion = GetNullableString(reader, reader.GetOrdinal("engine_version")),
BaselineDigest = GetNullableString(reader, reader.GetOrdinal("baseline_digest")),
ImageSigned = GetNullableBoolean(reader, reader.GetOrdinal("image_signed")),
SbomReferrer = GetNullableString(reader, reader.GetOrdinal("sbom_referrer")),
BuildId = GetNullableString(reader, reader.GetOrdinal("build_id")),
PayloadJson = reader.IsDBNull(payloadOrdinal) ? "{}" : reader.GetString(payloadOrdinal)
};
}
}
public readonly record struct RuntimeEventInsertResult(int InsertedCount, int DuplicateCount)
{
public static RuntimeEventInsertResult Empty => new(0, 0);
}
public sealed record RuntimeBuildIdObservation(
string ImageDigest,
IReadOnlyList<string> BuildIds,
DateTime ObservedAtUtc);

View File

@@ -1,144 +1,144 @@
using System.Buffers;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Services;
public sealed class ArtifactStorageService
{
private readonly ArtifactRepository _artifactRepository;
private readonly LifecycleRuleRepository _lifecycleRuleRepository;
private readonly IArtifactObjectStore _objectStore;
private readonly ScannerStorageOptions _options;
private readonly ILogger<ArtifactStorageService> _logger;
private readonly TimeProvider _timeProvider;
public ArtifactStorageService(
ArtifactRepository artifactRepository,
LifecycleRuleRepository lifecycleRuleRepository,
IArtifactObjectStore objectStore,
IOptions<ScannerStorageOptions> options,
ILogger<ArtifactStorageService> logger,
TimeProvider? timeProvider = null)
{
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
_lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<ArtifactDocument> StoreArtifactAsync(
ArtifactDocumentType type,
ArtifactDocumentFormat format,
string mediaType,
Stream content,
bool immutable,
string ttlClass,
DateTime? expiresAtUtc,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(content);
ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false);
try
{
var normalizedDigest = $"sha256:{digestHex}";
using System.Buffers;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Services;
public sealed class ArtifactStorageService
{
private readonly ArtifactRepository _artifactRepository;
private readonly LifecycleRuleRepository _lifecycleRuleRepository;
private readonly IArtifactObjectStore _objectStore;
private readonly ScannerStorageOptions _options;
private readonly ILogger<ArtifactStorageService> _logger;
private readonly TimeProvider _timeProvider;
public ArtifactStorageService(
ArtifactRepository artifactRepository,
LifecycleRuleRepository lifecycleRuleRepository,
IArtifactObjectStore objectStore,
IOptions<ScannerStorageOptions> options,
ILogger<ArtifactStorageService> logger,
TimeProvider? timeProvider = null)
{
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
_lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<ArtifactDocument> StoreArtifactAsync(
ArtifactDocumentType type,
ArtifactDocumentFormat format,
string mediaType,
Stream content,
bool immutable,
string ttlClass,
DateTime? expiresAtUtc,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(content);
ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false);
try
{
var normalizedDigest = $"sha256:{digestHex}";
var artifactId = CatalogIdFactory.CreateArtifactId(type, normalizedDigest);
var key = ArtifactObjectKeyBuilder.Build(
type,
format,
normalizedDigest,
_options.ObjectStore.RootPrefix);
var descriptor = new ArtifactObjectDescriptor(
_options.ObjectStore.BucketName,
key,
immutable,
_options.ObjectStore.ComplianceRetention);
buffer.Position = 0;
await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false);
if (_options.DualWrite.Enabled)
{
buffer.Position = 0;
var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! };
await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false);
}
var now = _timeProvider.GetUtcNow().UtcDateTime;
var document = new ArtifactDocument
{
Id = artifactId,
Type = type,
Format = format,
MediaType = mediaType,
BytesSha256 = normalizedDigest,
SizeBytes = size,
Immutable = immutable,
RefCount = 1,
CreatedAtUtc = now,
UpdatedAtUtc = now,
TtlClass = ttlClass,
};
await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
if (expiresAtUtc.HasValue)
{
var lifecycle = new LifecycleRuleDocument
{
Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass),
ArtifactId = document.Id,
Class = ttlClass,
ExpiresAtUtc = expiresAtUtc,
CreatedAtUtc = now,
};
await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest);
return document;
}
finally
{
await buffer.DisposeAsync().ConfigureAwait(false);
}
}
private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken)
{
var bufferStream = new MemoryStream();
var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var rented = ArrayPool<byte>.Shared.Rent(81920);
long total = 0;
try
{
int read;
while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
hasher.AppendData(rented, 0, read);
await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
total += read;
}
}
finally
{
ArrayPool<byte>.Shared.Return(rented);
}
bufferStream.Position = 0;
var digest = hasher.GetCurrentHash();
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
return (bufferStream, total, digestHex);
}
var descriptor = new ArtifactObjectDescriptor(
_options.ObjectStore.BucketName,
key,
immutable,
_options.ObjectStore.ComplianceRetention);
buffer.Position = 0;
await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false);
if (_options.DualWrite.Enabled)
{
buffer.Position = 0;
var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! };
await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false);
}
var now = _timeProvider.GetUtcNow().UtcDateTime;
var document = new ArtifactDocument
{
Id = artifactId,
Type = type,
Format = format,
MediaType = mediaType,
BytesSha256 = normalizedDigest,
SizeBytes = size,
Immutable = immutable,
RefCount = 1,
CreatedAtUtc = now,
UpdatedAtUtc = now,
TtlClass = ttlClass,
};
await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
if (expiresAtUtc.HasValue)
{
var lifecycle = new LifecycleRuleDocument
{
Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass),
ArtifactId = document.Id,
Class = ttlClass,
ExpiresAtUtc = expiresAtUtc,
CreatedAtUtc = now,
};
await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest);
return document;
}
finally
{
await buffer.DisposeAsync().ConfigureAwait(false);
}
}
private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken)
{
var bufferStream = new MemoryStream();
var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var rented = ArrayPool<byte>.Shared.Rent(81920);
long total = 0;
try
{
int read;
while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
hasher.AppendData(rented, 0, read);
await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
total += read;
}
}
finally
{
ArrayPool<byte>.Shared.Return(rented);
}
bufferStream.Position = 0;
var digest = hasher.GetCurrentHash();
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
return (bufferStream, total, digestHex);
}
}