up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,43 +1,43 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public static class CatalogIdFactory
|
||||
{
|
||||
public static string CreateArtifactId(ArtifactDocumentType type, string digest)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
|
||||
return $"{type.ToString().ToLowerInvariant()}::{NormalizeDigest(digest)}";
|
||||
}
|
||||
|
||||
public static string CreateLinkId(LinkSourceType type, string fromDigest, string artifactId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(fromDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
|
||||
var input = Encoding.UTF8.GetBytes($"{type}:{NormalizeDigest(fromDigest)}:{artifactId}");
|
||||
var hash = SHA256.HashData(input);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public static string CreateLifecycleRuleId(string artifactId, string @class)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
var normalizedClass = string.IsNullOrWhiteSpace(@class) ? "default" : @class.Trim().ToLowerInvariant();
|
||||
var payload = Encoding.UTF8.GetBytes($"{artifactId}:{normalizedClass}");
|
||||
var hash = SHA256.HashData(payload);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizeDigest(string digest)
|
||||
{
|
||||
if (!digest.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
return $"sha256:{digest.Trim().ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
var parts = digest.Split(':', 2, StringSplitOptions.TrimEntries);
|
||||
return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}";
|
||||
}
|
||||
}
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public static class CatalogIdFactory
|
||||
{
|
||||
public static string CreateArtifactId(ArtifactDocumentType type, string digest)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
|
||||
return $"{type.ToString().ToLowerInvariant()}::{NormalizeDigest(digest)}";
|
||||
}
|
||||
|
||||
public static string CreateLinkId(LinkSourceType type, string fromDigest, string artifactId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(fromDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
|
||||
var input = Encoding.UTF8.GetBytes($"{type}:{NormalizeDigest(fromDigest)}:{artifactId}");
|
||||
var hash = SHA256.HashData(input);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public static string CreateLifecycleRuleId(string artifactId, string @class)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
var normalizedClass = string.IsNullOrWhiteSpace(@class) ? "default" : @class.Trim().ToLowerInvariant();
|
||||
var payload = Encoding.UTF8.GetBytes($"{artifactId}:{normalizedClass}");
|
||||
var hash = SHA256.HashData(payload);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizeDigest(string digest)
|
||||
{
|
||||
if (!digest.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
return $"sha256:{digest.Trim().ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
var parts = digest.Split(':', 2, StringSplitOptions.TrimEntries);
|
||||
return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public sealed class ImageDocument
|
||||
{
|
||||
public string ImageDigest { get; set; } = string.Empty;
|
||||
|
||||
public string Repository { get; set; } = string.Empty;
|
||||
|
||||
public string? Tag { get; set; }
|
||||
= null;
|
||||
|
||||
public string Architecture { get; set; } = string.Empty;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
public DateTime LastSeenAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public sealed class ImageDocument
|
||||
{
|
||||
public string ImageDigest { get; set; } = string.Empty;
|
||||
|
||||
public string Repository { get; set; } = string.Empty;
|
||||
|
||||
public string? Tag { get; set; }
|
||||
= null;
|
||||
|
||||
public string Architecture { get; set; } = string.Empty;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
public DateTime LastSeenAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
|
||||
@@ -1,37 +1,37 @@
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public enum JobState
|
||||
{
|
||||
Pending,
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
public sealed class JobDocument
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
|
||||
public JobState State { get; set; } = JobState.Pending;
|
||||
|
||||
public string ArgumentsJson { get; set; }
|
||||
= "{}";
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
public DateTime? StartedAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime? CompletedAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime? HeartbeatAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Error { get; set; }
|
||||
= null;
|
||||
}
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public enum JobState
|
||||
{
|
||||
Pending,
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
public sealed class JobDocument
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
|
||||
public JobState State { get; set; } = JobState.Pending;
|
||||
|
||||
public string ArgumentsJson { get; set; }
|
||||
= "{}";
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
public DateTime? StartedAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime? CompletedAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime? HeartbeatAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Error { get; set; }
|
||||
= null;
|
||||
}
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public sealed class LayerDocument
|
||||
{
|
||||
public string LayerDigest { get; set; } = string.Empty;
|
||||
|
||||
public string MediaType { get; set; } = string.Empty;
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
= 0;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
public DateTime LastSeenAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public sealed class LayerDocument
|
||||
{
|
||||
public string LayerDigest { get; set; } = string.Empty;
|
||||
|
||||
public string MediaType { get; set; } = string.Empty;
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
= 0;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
public DateTime LastSeenAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public sealed class LifecycleRuleDocument
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public string ArtifactId { get; set; } = string.Empty;
|
||||
|
||||
public string Class { get; set; } = "default";
|
||||
|
||||
public DateTime? ExpiresAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public sealed class LifecycleRuleDocument
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public string ArtifactId { get; set; } = string.Empty;
|
||||
|
||||
public string Class { get; set; } = "default";
|
||||
|
||||
public DateTime? ExpiresAtUtc { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public enum LinkSourceType
|
||||
{
|
||||
Image,
|
||||
Layer,
|
||||
}
|
||||
|
||||
public sealed class LinkDocument
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public LinkSourceType FromType { get; set; }
|
||||
= LinkSourceType.Image;
|
||||
|
||||
public string FromDigest { get; set; } = string.Empty;
|
||||
|
||||
public string ArtifactId { get; set; } = string.Empty;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
public enum LinkSourceType
|
||||
{
|
||||
Image,
|
||||
Layer,
|
||||
}
|
||||
|
||||
public sealed class LinkDocument
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
public LinkSourceType FromType { get; set; }
|
||||
= LinkSourceType.Image;
|
||||
|
||||
public string FromDigest { get; set; } = string.Empty;
|
||||
|
||||
public string ArtifactId { get; set; } = string.Empty;
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
}
|
||||
|
||||
@@ -1,53 +1,53 @@
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
/// <summary>
|
||||
/// Persistence model for runtime events emitted by the Zastava observer.
|
||||
/// </summary>
|
||||
public sealed class RuntimeEventDocument
|
||||
{
|
||||
public string? Id { get; set; }
|
||||
|
||||
public string EventId { get; set; } = string.Empty;
|
||||
|
||||
public string SchemaVersion { get; set; } = string.Empty;
|
||||
|
||||
public string Tenant { get; set; } = string.Empty;
|
||||
|
||||
public string Node { get; set; } = string.Empty;
|
||||
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
|
||||
public DateTime When { get; set; }
|
||||
|
||||
public DateTime ReceivedAt { get; set; }
|
||||
|
||||
public DateTime ExpiresAt { get; set; }
|
||||
|
||||
public string? Platform { get; set; }
|
||||
|
||||
public string? Namespace { get; set; }
|
||||
|
||||
public string? Pod { get; set; }
|
||||
|
||||
public string? Container { get; set; }
|
||||
|
||||
public string? ContainerId { get; set; }
|
||||
|
||||
public string? ImageRef { get; set; }
|
||||
|
||||
public string? ImageDigest { get; set; }
|
||||
|
||||
public string? Engine { get; set; }
|
||||
|
||||
public string? EngineVersion { get; set; }
|
||||
|
||||
public string? BaselineDigest { get; set; }
|
||||
|
||||
public bool? ImageSigned { get; set; }
|
||||
|
||||
public string? SbomReferrer { get; set; }
|
||||
|
||||
public string? BuildId { get; set; }
|
||||
|
||||
public string PayloadJson { get; set; } = "{}";
|
||||
}
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
/// <summary>
|
||||
/// Persistence model for runtime events emitted by the Zastava observer.
|
||||
/// </summary>
|
||||
public sealed class RuntimeEventDocument
|
||||
{
|
||||
public string? Id { get; set; }
|
||||
|
||||
public string EventId { get; set; } = string.Empty;
|
||||
|
||||
public string SchemaVersion { get; set; } = string.Empty;
|
||||
|
||||
public string Tenant { get; set; } = string.Empty;
|
||||
|
||||
public string Node { get; set; } = string.Empty;
|
||||
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
|
||||
public DateTime When { get; set; }
|
||||
|
||||
public DateTime ReceivedAt { get; set; }
|
||||
|
||||
public DateTime ExpiresAt { get; set; }
|
||||
|
||||
public string? Platform { get; set; }
|
||||
|
||||
public string? Namespace { get; set; }
|
||||
|
||||
public string? Pod { get; set; }
|
||||
|
||||
public string? Container { get; set; }
|
||||
|
||||
public string? ContainerId { get; set; }
|
||||
|
||||
public string? ImageRef { get; set; }
|
||||
|
||||
public string? ImageDigest { get; set; }
|
||||
|
||||
public string? Engine { get; set; }
|
||||
|
||||
public string? EngineVersion { get; set; }
|
||||
|
||||
public string? BaselineDigest { get; set; }
|
||||
|
||||
public bool? ImageSigned { get; set; }
|
||||
|
||||
public string? SbomReferrer { get; set; }
|
||||
|
||||
public string? BuildId { get; set; }
|
||||
|
||||
public string PayloadJson { get; set; } = "{}";
|
||||
}
|
||||
|
||||
@@ -1,173 +1,173 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using Amazon;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Infrastructure.Postgres.Migrations;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Storage.Services;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Extensions;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddScannerStorage(this IServiceCollection services, Action<ScannerStorageOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<ScannerStorageOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(options => options.EnsureValid())
|
||||
.ValidateOnStart();
|
||||
|
||||
RegisterScannerStorageServices(services);
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddScannerStorage(this IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddOptions<ScannerStorageOptions>()
|
||||
.Bind(configuration)
|
||||
.PostConfigure(options => options.EnsureValid())
|
||||
.ValidateOnStart();
|
||||
|
||||
RegisterScannerStorageServices(services);
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void RegisterScannerStorageServices(IServiceCollection services)
|
||||
{
|
||||
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
|
||||
services.TryAddSingleton<ScannerDataSource>();
|
||||
|
||||
services.AddStartupMigrations<ScannerStorageOptions>(
|
||||
ScannerDataSource.DefaultSchema,
|
||||
"Scanner.Storage",
|
||||
typeof(ScannerDataSource).Assembly,
|
||||
options => options.Postgres.ConnectionString);
|
||||
|
||||
services.AddMigrationStatus<ScannerStorageOptions>(
|
||||
ScannerDataSource.DefaultSchema,
|
||||
"Scanner.Storage",
|
||||
typeof(ScannerDataSource).Assembly,
|
||||
options => options.Postgres.ConnectionString);
|
||||
|
||||
services.AddScoped<ArtifactRepository>();
|
||||
services.AddScoped<ImageRepository>();
|
||||
services.AddScoped<LayerRepository>();
|
||||
services.AddScoped<LinkRepository>();
|
||||
services.AddScoped<JobRepository>();
|
||||
services.AddScoped<LifecycleRuleRepository>();
|
||||
services.AddScoped<RuntimeEventRepository>();
|
||||
services.AddScoped<EntryTraceRepository>();
|
||||
services.AddScoped<RubyPackageInventoryRepository>();
|
||||
services.AddScoped<BunPackageInventoryRepository>();
|
||||
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
|
||||
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
|
||||
services.AddSingleton<IBunPackageInventoryStore, BunPackageInventoryStore>();
|
||||
|
||||
services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName)
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
|
||||
if (!options.IsRustFsDriver())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(options.RustFs.BaseUrl, UriKind.Absolute, out var baseUri))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS baseUrl must be a valid absolute URI.");
|
||||
}
|
||||
|
||||
client.BaseAddress = baseUri;
|
||||
client.Timeout = options.RustFs.Timeout;
|
||||
|
||||
foreach (var header in options.Headers)
|
||||
{
|
||||
client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.RustFs.ApiKeyHeader)
|
||||
&& !string.IsNullOrWhiteSpace(options.RustFs.ApiKey))
|
||||
{
|
||||
client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFs.ApiKeyHeader, options.RustFs.ApiKey);
|
||||
}
|
||||
})
|
||||
.ConfigurePrimaryHttpMessageHandler(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
|
||||
if (!options.IsRustFsDriver())
|
||||
{
|
||||
return new HttpClientHandler();
|
||||
}
|
||||
|
||||
var handler = new HttpClientHandler();
|
||||
if (options.RustFs.AllowInsecureTls)
|
||||
{
|
||||
handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
|
||||
}
|
||||
|
||||
return handler;
|
||||
});
|
||||
|
||||
services.TryAddSingleton(CreateAmazonS3Client);
|
||||
services.TryAddSingleton<IArtifactObjectStore>(CreateArtifactObjectStore);
|
||||
services.TryAddSingleton<ArtifactStorageService>();
|
||||
}
|
||||
|
||||
private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider)
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
|
||||
var config = new AmazonS3Config
|
||||
{
|
||||
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
|
||||
ForcePathStyle = options.ForcePathStyle,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
|
||||
{
|
||||
config.ServiceURL = options.ServiceUrl;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.AccessKeyId) && !string.IsNullOrWhiteSpace(options.SecretAccessKey))
|
||||
{
|
||||
AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken)
|
||||
? new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey)
|
||||
: new SessionAWSCredentials(options.AccessKeyId, options.SecretAccessKey, options.SessionToken);
|
||||
return new AmazonS3Client(credentials, config);
|
||||
}
|
||||
|
||||
return new AmazonS3Client(config);
|
||||
}
|
||||
|
||||
private static IArtifactObjectStore CreateArtifactObjectStore(IServiceProvider provider)
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>();
|
||||
var objectStore = options.Value.ObjectStore;
|
||||
|
||||
if (objectStore.IsRustFsDriver())
|
||||
{
|
||||
return new RustFsArtifactObjectStore(
|
||||
provider.GetRequiredService<IHttpClientFactory>(),
|
||||
options,
|
||||
provider.GetRequiredService<ILogger<RustFsArtifactObjectStore>>());
|
||||
}
|
||||
|
||||
return new S3ArtifactObjectStore(
|
||||
provider.GetRequiredService<IAmazonS3>(),
|
||||
options,
|
||||
provider.GetRequiredService<ILogger<S3ArtifactObjectStore>>());
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using Amazon;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Infrastructure.Postgres.Migrations;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Storage.Services;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Extensions;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddScannerStorage(this IServiceCollection services, Action<ScannerStorageOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<ScannerStorageOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(options => options.EnsureValid())
|
||||
.ValidateOnStart();
|
||||
|
||||
RegisterScannerStorageServices(services);
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddScannerStorage(this IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddOptions<ScannerStorageOptions>()
|
||||
.Bind(configuration)
|
||||
.PostConfigure(options => options.EnsureValid())
|
||||
.ValidateOnStart();
|
||||
|
||||
RegisterScannerStorageServices(services);
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void RegisterScannerStorageServices(IServiceCollection services)
|
||||
{
|
||||
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
|
||||
services.TryAddSingleton<ScannerDataSource>();
|
||||
|
||||
services.AddStartupMigrations<ScannerStorageOptions>(
|
||||
ScannerDataSource.DefaultSchema,
|
||||
"Scanner.Storage",
|
||||
typeof(ScannerDataSource).Assembly,
|
||||
options => options.Postgres.ConnectionString);
|
||||
|
||||
services.AddMigrationStatus<ScannerStorageOptions>(
|
||||
ScannerDataSource.DefaultSchema,
|
||||
"Scanner.Storage",
|
||||
typeof(ScannerDataSource).Assembly,
|
||||
options => options.Postgres.ConnectionString);
|
||||
|
||||
services.AddScoped<ArtifactRepository>();
|
||||
services.AddScoped<ImageRepository>();
|
||||
services.AddScoped<LayerRepository>();
|
||||
services.AddScoped<LinkRepository>();
|
||||
services.AddScoped<JobRepository>();
|
||||
services.AddScoped<LifecycleRuleRepository>();
|
||||
services.AddScoped<RuntimeEventRepository>();
|
||||
services.AddScoped<EntryTraceRepository>();
|
||||
services.AddScoped<RubyPackageInventoryRepository>();
|
||||
services.AddScoped<BunPackageInventoryRepository>();
|
||||
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
|
||||
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
|
||||
services.AddSingleton<IBunPackageInventoryStore, BunPackageInventoryStore>();
|
||||
|
||||
services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName)
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
|
||||
if (!options.IsRustFsDriver())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(options.RustFs.BaseUrl, UriKind.Absolute, out var baseUri))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS baseUrl must be a valid absolute URI.");
|
||||
}
|
||||
|
||||
client.BaseAddress = baseUri;
|
||||
client.Timeout = options.RustFs.Timeout;
|
||||
|
||||
foreach (var header in options.Headers)
|
||||
{
|
||||
client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.RustFs.ApiKeyHeader)
|
||||
&& !string.IsNullOrWhiteSpace(options.RustFs.ApiKey))
|
||||
{
|
||||
client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFs.ApiKeyHeader, options.RustFs.ApiKey);
|
||||
}
|
||||
})
|
||||
.ConfigurePrimaryHttpMessageHandler(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
|
||||
if (!options.IsRustFsDriver())
|
||||
{
|
||||
return new HttpClientHandler();
|
||||
}
|
||||
|
||||
var handler = new HttpClientHandler();
|
||||
if (options.RustFs.AllowInsecureTls)
|
||||
{
|
||||
handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
|
||||
}
|
||||
|
||||
return handler;
|
||||
});
|
||||
|
||||
services.TryAddSingleton(CreateAmazonS3Client);
|
||||
services.TryAddSingleton<IArtifactObjectStore>(CreateArtifactObjectStore);
|
||||
services.TryAddSingleton<ArtifactStorageService>();
|
||||
}
|
||||
|
||||
private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider)
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;
|
||||
var config = new AmazonS3Config
|
||||
{
|
||||
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
|
||||
ForcePathStyle = options.ForcePathStyle,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
|
||||
{
|
||||
config.ServiceURL = options.ServiceUrl;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.AccessKeyId) && !string.IsNullOrWhiteSpace(options.SecretAccessKey))
|
||||
{
|
||||
AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken)
|
||||
? new BasicAWSCredentials(options.AccessKeyId, options.SecretAccessKey)
|
||||
: new SessionAWSCredentials(options.AccessKeyId, options.SecretAccessKey, options.SessionToken);
|
||||
return new AmazonS3Client(credentials, config);
|
||||
}
|
||||
|
||||
return new AmazonS3Client(config);
|
||||
}
|
||||
|
||||
private static IArtifactObjectStore CreateArtifactObjectStore(IServiceProvider provider)
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>();
|
||||
var objectStore = options.Value.ObjectStore;
|
||||
|
||||
if (objectStore.IsRustFsDriver())
|
||||
{
|
||||
return new RustFsArtifactObjectStore(
|
||||
provider.GetRequiredService<IHttpClientFactory>(),
|
||||
options,
|
||||
provider.GetRequiredService<ILogger<RustFsArtifactObjectStore>>());
|
||||
}
|
||||
|
||||
return new S3ArtifactObjectStore(
|
||||
provider.GetRequiredService<IAmazonS3>(),
|
||||
options,
|
||||
provider.GetRequiredService<ILogger<S3ArtifactObjectStore>>());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
namespace StellaOps.Scanner.Storage.ObjectStore;
|
||||
|
||||
public interface IArtifactObjectStore
|
||||
{
|
||||
Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken);
|
||||
|
||||
Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
|
||||
|
||||
Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ArtifactObjectDescriptor(string Bucket, string Key, bool Immutable, TimeSpan? RetainFor = null);
|
||||
namespace StellaOps.Scanner.Storage.ObjectStore;
|
||||
|
||||
public interface IArtifactObjectStore
|
||||
{
|
||||
Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken);
|
||||
|
||||
Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
|
||||
|
||||
Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ArtifactObjectDescriptor(string Bucket, string Key, bool Immutable, TimeSpan? RetainFor = null);
|
||||
|
||||
@@ -1,75 +1,75 @@
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.ObjectStore;
|
||||
|
||||
public sealed class S3ArtifactObjectStore : IArtifactObjectStore
|
||||
{
|
||||
private readonly IAmazonS3 _s3;
|
||||
private readonly ObjectStoreOptions _options;
|
||||
private readonly ILogger<S3ArtifactObjectStore> _logger;
|
||||
|
||||
public S3ArtifactObjectStore(IAmazonS3 s3, IOptions<ScannerStorageOptions> options, ILogger<S3ArtifactObjectStore> logger)
|
||||
{
|
||||
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.ObjectStore;
|
||||
}
|
||||
|
||||
public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(descriptor);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var request = new PutObjectRequest
|
||||
{
|
||||
BucketName = descriptor.Bucket,
|
||||
Key = descriptor.Key,
|
||||
InputStream = content,
|
||||
AutoCloseStream = false,
|
||||
};
|
||||
|
||||
if (descriptor.Immutable && _options.EnableObjectLock)
|
||||
{
|
||||
request.ObjectLockMode = ObjectLockMode.Compliance;
|
||||
if (descriptor.RetainFor is { } retention && retention > TimeSpan.Zero)
|
||||
{
|
||||
request.ObjectLockRetainUntilDate = DateTime.UtcNow + retention;
|
||||
}
|
||||
else if (_options.ComplianceRetention is { } defaultRetention && defaultRetention > TimeSpan.Zero)
|
||||
{
|
||||
request.ObjectLockRetainUntilDate = DateTime.UtcNow + defaultRetention;
|
||||
}
|
||||
}
|
||||
|
||||
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("Uploaded scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
|
||||
}
|
||||
|
||||
public async Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(descriptor);
|
||||
try
|
||||
{
|
||||
var response = await _s3.GetObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
|
||||
var buffer = new MemoryStream();
|
||||
await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
return buffer;
|
||||
}
|
||||
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
{
|
||||
_logger.LogDebug("Scanner object {Bucket}/{Key} not found", descriptor.Bucket, descriptor.Key);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(descriptor);
|
||||
await _s3.DeleteObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("Deleted scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
|
||||
}
|
||||
}
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.ObjectStore;
|
||||
|
||||
public sealed class S3ArtifactObjectStore : IArtifactObjectStore
|
||||
{
|
||||
private readonly IAmazonS3 _s3;
|
||||
private readonly ObjectStoreOptions _options;
|
||||
private readonly ILogger<S3ArtifactObjectStore> _logger;
|
||||
|
||||
public S3ArtifactObjectStore(IAmazonS3 s3, IOptions<ScannerStorageOptions> options, ILogger<S3ArtifactObjectStore> logger)
|
||||
{
|
||||
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.ObjectStore;
|
||||
}
|
||||
|
||||
public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(descriptor);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var request = new PutObjectRequest
|
||||
{
|
||||
BucketName = descriptor.Bucket,
|
||||
Key = descriptor.Key,
|
||||
InputStream = content,
|
||||
AutoCloseStream = false,
|
||||
};
|
||||
|
||||
if (descriptor.Immutable && _options.EnableObjectLock)
|
||||
{
|
||||
request.ObjectLockMode = ObjectLockMode.Compliance;
|
||||
if (descriptor.RetainFor is { } retention && retention > TimeSpan.Zero)
|
||||
{
|
||||
request.ObjectLockRetainUntilDate = DateTime.UtcNow + retention;
|
||||
}
|
||||
else if (_options.ComplianceRetention is { } defaultRetention && defaultRetention > TimeSpan.Zero)
|
||||
{
|
||||
request.ObjectLockRetainUntilDate = DateTime.UtcNow + defaultRetention;
|
||||
}
|
||||
}
|
||||
|
||||
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("Uploaded scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
|
||||
}
|
||||
|
||||
public async Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(descriptor);
|
||||
try
|
||||
{
|
||||
var response = await _s3.GetObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
|
||||
var buffer = new MemoryStream();
|
||||
await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
return buffer;
|
||||
}
|
||||
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
{
|
||||
_logger.LogDebug("Scanner object {Bucket}/{Key} not found", descriptor.Bucket, descriptor.Key);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(descriptor);
|
||||
await _s3.DeleteObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("Deleted scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,176 +1,176 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class ArtifactRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.artifacts";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ArtifactRepository(
|
||||
ScannerDataSource dataSource,
|
||||
ILogger<ArtifactRepository> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task<ArtifactDocument?> GetAsync(string artifactId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
|
||||
rekor, ttl_class, created_at_utc, updated_at_utc
|
||||
FROM {Table}
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "id", artifactId),
|
||||
MapArtifact,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
document.UpdatedAtUtc = now;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
|
||||
rekor, ttl_class, created_at_utc, updated_at_utc
|
||||
)
|
||||
VALUES (
|
||||
@id, @type, @format, @media_type, @bytes_sha256, @size_bytes, @immutable, @ref_count,
|
||||
@rekor::jsonb, @ttl_class, @created_at_utc, @updated_at_utc
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
format = EXCLUDED.format,
|
||||
media_type = EXCLUDED.media_type,
|
||||
bytes_sha256 = EXCLUDED.bytes_sha256,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
immutable = EXCLUDED.immutable,
|
||||
ref_count = EXCLUDED.ref_count,
|
||||
rekor = EXCLUDED.rekor,
|
||||
ttl_class = EXCLUDED.ttl_class,
|
||||
updated_at_utc = EXCLUDED.updated_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "type", document.Type.ToString());
|
||||
AddParameter(cmd, "format", document.Format.ToString());
|
||||
AddParameter(cmd, "media_type", document.MediaType);
|
||||
AddParameter(cmd, "bytes_sha256", document.BytesSha256);
|
||||
AddParameter(cmd, "size_bytes", document.SizeBytes);
|
||||
AddParameter(cmd, "immutable", document.Immutable);
|
||||
AddParameter(cmd, "ref_count", document.RefCount);
|
||||
AddJsonbParameter(cmd, "rekor", SerializeRekor(document.Rekor));
|
||||
AddParameter(cmd, "ttl_class", document.TtlClass);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "updated_at_utc", document.UpdatedAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
ArgumentNullException.ThrowIfNull(reference);
|
||||
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
|
||||
var sql = $"""
|
||||
UPDATE {Table}
|
||||
SET rekor = @rekor::jsonb,
|
||||
updated_at_utc = @updated_at_utc
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", artifactId);
|
||||
AddJsonbParameter(cmd, "rekor", SerializeRekor(reference));
|
||||
AddParameter(cmd, "updated_at_utc", now);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<long> IncrementRefCountAsync(string artifactId, long delta, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
|
||||
var sql = $"""
|
||||
UPDATE {Table}
|
||||
SET ref_count = ref_count + @delta,
|
||||
updated_at_utc = NOW() AT TIME ZONE 'UTC'
|
||||
WHERE id = @id
|
||||
RETURNING ref_count
|
||||
""";
|
||||
|
||||
var result = await ExecuteScalarAsync<long?>(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", artifactId);
|
||||
AddParameter(cmd, "delta", delta);
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return result ?? 0;
|
||||
}
|
||||
|
||||
private static ArtifactDocument MapArtifact(NpgsqlDataReader reader)
|
||||
{
|
||||
var rekorOrdinal = reader.GetOrdinal("rekor");
|
||||
return new ArtifactDocument
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
Type = ParseEnum(reader.GetString(reader.GetOrdinal("type")), ArtifactDocumentType.ImageBom),
|
||||
Format = ParseEnum(reader.GetString(reader.GetOrdinal("format")), ArtifactDocumentFormat.CycloneDxJson),
|
||||
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
|
||||
BytesSha256 = reader.GetString(reader.GetOrdinal("bytes_sha256")),
|
||||
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
|
||||
Immutable = reader.GetBoolean(reader.GetOrdinal("immutable")),
|
||||
RefCount = reader.GetInt64(reader.GetOrdinal("ref_count")),
|
||||
Rekor = reader.IsDBNull(rekorOrdinal)
|
||||
? null
|
||||
: JsonSerializer.Deserialize<RekorReference>(reader.GetString(rekorOrdinal), JsonOptions),
|
||||
TtlClass = reader.GetString(reader.GetOrdinal("ttl_class")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
UpdatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("updated_at_utc")),
|
||||
};
|
||||
}
|
||||
|
||||
private static TEnum ParseEnum<TEnum>(string value, TEnum fallback) where TEnum : struct
|
||||
=> Enum.TryParse<TEnum>(value, ignoreCase: true, out var parsed) ? parsed : fallback;
|
||||
|
||||
private static string? SerializeRekor(RekorReference? reference)
|
||||
=> reference is null ? null : JsonSerializer.Serialize(reference, JsonOptions);
|
||||
}
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class ArtifactRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.artifacts";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ArtifactRepository(
|
||||
ScannerDataSource dataSource,
|
||||
ILogger<ArtifactRepository> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task<ArtifactDocument?> GetAsync(string artifactId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
|
||||
rekor, ttl_class, created_at_utc, updated_at_utc
|
||||
FROM {Table}
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "id", artifactId),
|
||||
MapArtifact,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
document.UpdatedAtUtc = now;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
|
||||
rekor, ttl_class, created_at_utc, updated_at_utc
|
||||
)
|
||||
VALUES (
|
||||
@id, @type, @format, @media_type, @bytes_sha256, @size_bytes, @immutable, @ref_count,
|
||||
@rekor::jsonb, @ttl_class, @created_at_utc, @updated_at_utc
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
format = EXCLUDED.format,
|
||||
media_type = EXCLUDED.media_type,
|
||||
bytes_sha256 = EXCLUDED.bytes_sha256,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
immutable = EXCLUDED.immutable,
|
||||
ref_count = EXCLUDED.ref_count,
|
||||
rekor = EXCLUDED.rekor,
|
||||
ttl_class = EXCLUDED.ttl_class,
|
||||
updated_at_utc = EXCLUDED.updated_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "type", document.Type.ToString());
|
||||
AddParameter(cmd, "format", document.Format.ToString());
|
||||
AddParameter(cmd, "media_type", document.MediaType);
|
||||
AddParameter(cmd, "bytes_sha256", document.BytesSha256);
|
||||
AddParameter(cmd, "size_bytes", document.SizeBytes);
|
||||
AddParameter(cmd, "immutable", document.Immutable);
|
||||
AddParameter(cmd, "ref_count", document.RefCount);
|
||||
AddJsonbParameter(cmd, "rekor", SerializeRekor(document.Rekor));
|
||||
AddParameter(cmd, "ttl_class", document.TtlClass);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "updated_at_utc", document.UpdatedAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
ArgumentNullException.ThrowIfNull(reference);
|
||||
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
|
||||
var sql = $"""
|
||||
UPDATE {Table}
|
||||
SET rekor = @rekor::jsonb,
|
||||
updated_at_utc = @updated_at_utc
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", artifactId);
|
||||
AddJsonbParameter(cmd, "rekor", SerializeRekor(reference));
|
||||
AddParameter(cmd, "updated_at_utc", now);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<long> IncrementRefCountAsync(string artifactId, long delta, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
|
||||
|
||||
var sql = $"""
|
||||
UPDATE {Table}
|
||||
SET ref_count = ref_count + @delta,
|
||||
updated_at_utc = NOW() AT TIME ZONE 'UTC'
|
||||
WHERE id = @id
|
||||
RETURNING ref_count
|
||||
""";
|
||||
|
||||
var result = await ExecuteScalarAsync<long?>(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", artifactId);
|
||||
AddParameter(cmd, "delta", delta);
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return result ?? 0;
|
||||
}
|
||||
|
||||
private static ArtifactDocument MapArtifact(NpgsqlDataReader reader)
|
||||
{
|
||||
var rekorOrdinal = reader.GetOrdinal("rekor");
|
||||
return new ArtifactDocument
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
Type = ParseEnum(reader.GetString(reader.GetOrdinal("type")), ArtifactDocumentType.ImageBom),
|
||||
Format = ParseEnum(reader.GetString(reader.GetOrdinal("format")), ArtifactDocumentFormat.CycloneDxJson),
|
||||
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
|
||||
BytesSha256 = reader.GetString(reader.GetOrdinal("bytes_sha256")),
|
||||
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
|
||||
Immutable = reader.GetBoolean(reader.GetOrdinal("immutable")),
|
||||
RefCount = reader.GetInt64(reader.GetOrdinal("ref_count")),
|
||||
Rekor = reader.IsDBNull(rekorOrdinal)
|
||||
? null
|
||||
: JsonSerializer.Deserialize<RekorReference>(reader.GetString(rekorOrdinal), JsonOptions),
|
||||
TtlClass = reader.GetString(reader.GetOrdinal("ttl_class")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
UpdatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("updated_at_utc")),
|
||||
};
|
||||
}
|
||||
|
||||
private static TEnum ParseEnum<TEnum>(string value, TEnum fallback) where TEnum : struct
|
||||
=> Enum.TryParse<TEnum>(value, ignoreCase: true, out var parsed) ? parsed : fallback;
|
||||
|
||||
private static string? SerializeRekor(RekorReference? reference)
|
||||
=> reference is null ? null : JsonSerializer.Serialize(reference, JsonOptions);
|
||||
}
|
||||
|
||||
@@ -1,86 +1,86 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class ImageRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.images";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ImageRepository(ScannerDataSource dataSource, ILogger<ImageRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.LastSeenAtUtc = now;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
|
||||
)
|
||||
VALUES (
|
||||
@image_digest, @repository, @tag, @architecture, @created_at_utc, @last_seen_at_utc
|
||||
)
|
||||
ON CONFLICT (image_digest) DO UPDATE SET
|
||||
repository = EXCLUDED.repository,
|
||||
tag = EXCLUDED.tag,
|
||||
architecture = EXCLUDED.architecture,
|
||||
last_seen_at_utc = EXCLUDED.last_seen_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "image_digest", document.ImageDigest);
|
||||
AddParameter(cmd, "repository", document.Repository);
|
||||
AddParameter(cmd, "tag", document.Tag);
|
||||
AddParameter(cmd, "architecture", document.Architecture);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<ImageDocument?> GetAsync(string imageDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
|
||||
var sql = $"""
|
||||
SELECT image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
|
||||
FROM {Table}
|
||||
WHERE image_digest = @image_digest
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "image_digest", imageDigest),
|
||||
MapImage,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static ImageDocument MapImage(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
|
||||
Repository = reader.GetString(reader.GetOrdinal("repository")),
|
||||
Tag = GetNullableString(reader, reader.GetOrdinal("tag")),
|
||||
Architecture = reader.GetString(reader.GetOrdinal("architecture")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
|
||||
};
|
||||
}
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class ImageRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.images";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ImageRepository(ScannerDataSource dataSource, ILogger<ImageRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.LastSeenAtUtc = now;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
|
||||
)
|
||||
VALUES (
|
||||
@image_digest, @repository, @tag, @architecture, @created_at_utc, @last_seen_at_utc
|
||||
)
|
||||
ON CONFLICT (image_digest) DO UPDATE SET
|
||||
repository = EXCLUDED.repository,
|
||||
tag = EXCLUDED.tag,
|
||||
architecture = EXCLUDED.architecture,
|
||||
last_seen_at_utc = EXCLUDED.last_seen_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "image_digest", document.ImageDigest);
|
||||
AddParameter(cmd, "repository", document.Repository);
|
||||
AddParameter(cmd, "tag", document.Tag);
|
||||
AddParameter(cmd, "architecture", document.Architecture);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<ImageDocument?> GetAsync(string imageDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
|
||||
var sql = $"""
|
||||
SELECT image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
|
||||
FROM {Table}
|
||||
WHERE image_digest = @image_digest
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "image_digest", imageDigest),
|
||||
MapImage,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static ImageDocument MapImage(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
|
||||
Repository = reader.GetString(reader.GetOrdinal("repository")),
|
||||
Tag = GetNullableString(reader, reader.GetOrdinal("tag")),
|
||||
Architecture = reader.GetString(reader.GetOrdinal("architecture")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,151 +1,151 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class JobRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.jobs";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public JobRepository(ScannerDataSource dataSource, ILogger<JobRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task<JobDocument> InsertAsync(JobDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.CreatedAtUtc = now;
|
||||
document.HeartbeatAtUtc = now;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
)
|
||||
VALUES (
|
||||
@id, @kind, @state::job_state, @args::jsonb, @created_at_utc, @started_at_utc, @completed_at_utc, @heartbeat_at_utc, @error
|
||||
)
|
||||
RETURNING id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
""";
|
||||
|
||||
var inserted = await QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "kind", document.Kind);
|
||||
AddParameter(cmd, "state", document.State.ToString());
|
||||
AddJsonbParameter(cmd, "args", document.ArgumentsJson);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "started_at_utc", document.StartedAtUtc);
|
||||
AddParameter(cmd, "completed_at_utc", document.CompletedAtUtc);
|
||||
AddParameter(cmd, "heartbeat_at_utc", document.HeartbeatAtUtc);
|
||||
AddParameter(cmd, "error", document.Error);
|
||||
},
|
||||
MapJob,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return inserted ?? document;
|
||||
}
|
||||
|
||||
public async Task<bool> TryTransitionAsync(string jobId, JobState expected, JobState next, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
|
||||
var sql = $"""
|
||||
UPDATE {Table}
|
||||
SET state = @next::job_state,
|
||||
heartbeat_at_utc = @heartbeat,
|
||||
started_at_utc = CASE
|
||||
WHEN @next = 'Running' THEN COALESCE(started_at_utc, @heartbeat)
|
||||
ELSE started_at_utc END,
|
||||
completed_at_utc = CASE
|
||||
WHEN @next IN ('Succeeded','Failed','Cancelled') THEN @heartbeat
|
||||
ELSE completed_at_utc END
|
||||
WHERE id = @id AND state = @expected::job_state
|
||||
""";
|
||||
|
||||
var affected = await ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", jobId);
|
||||
AddParameter(cmd, "expected", expected.ToString());
|
||||
AddParameter(cmd, "next", next.ToString());
|
||||
AddParameter(cmd, "heartbeat", now);
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return affected == 1;
|
||||
}
|
||||
|
||||
public Task<JobDocument?> GetAsync(string jobId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
FROM {Table}
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "id", jobId),
|
||||
MapJob,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<List<JobDocument>> ListStaleAsync(TimeSpan heartbeatThreshold, CancellationToken cancellationToken)
|
||||
{
|
||||
if (heartbeatThreshold <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(heartbeatThreshold));
|
||||
}
|
||||
|
||||
var cutoff = _timeProvider.GetUtcNow().UtcDateTime - heartbeatThreshold;
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
FROM {Table}
|
||||
WHERE state = 'Running'::job_state
|
||||
AND heartbeat_at_utc < @cutoff
|
||||
ORDER BY heartbeat_at_utc
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "cutoff", cutoff),
|
||||
MapJob,
|
||||
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
|
||||
}
|
||||
|
||||
private static JobDocument MapJob(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
Kind = reader.GetString(reader.GetOrdinal("kind")),
|
||||
State = Enum.TryParse<JobState>(reader.GetString(reader.GetOrdinal("state")), true, out var state)
|
||||
? state
|
||||
: JobState.Pending,
|
||||
ArgumentsJson = reader.GetString(reader.GetOrdinal("args")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
StartedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("started_at_utc"))?.UtcDateTime,
|
||||
CompletedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("completed_at_utc"))?.UtcDateTime,
|
||||
HeartbeatAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("heartbeat_at_utc"))?.UtcDateTime,
|
||||
Error = GetNullableString(reader, reader.GetOrdinal("error"))
|
||||
};
|
||||
}
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class JobRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.jobs";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public JobRepository(ScannerDataSource dataSource, ILogger<JobRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task<JobDocument> InsertAsync(JobDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.CreatedAtUtc = now;
|
||||
document.HeartbeatAtUtc = now;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
)
|
||||
VALUES (
|
||||
@id, @kind, @state::job_state, @args::jsonb, @created_at_utc, @started_at_utc, @completed_at_utc, @heartbeat_at_utc, @error
|
||||
)
|
||||
RETURNING id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
""";
|
||||
|
||||
var inserted = await QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "kind", document.Kind);
|
||||
AddParameter(cmd, "state", document.State.ToString());
|
||||
AddJsonbParameter(cmd, "args", document.ArgumentsJson);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "started_at_utc", document.StartedAtUtc);
|
||||
AddParameter(cmd, "completed_at_utc", document.CompletedAtUtc);
|
||||
AddParameter(cmd, "heartbeat_at_utc", document.HeartbeatAtUtc);
|
||||
AddParameter(cmd, "error", document.Error);
|
||||
},
|
||||
MapJob,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return inserted ?? document;
|
||||
}
|
||||
|
||||
public async Task<bool> TryTransitionAsync(string jobId, JobState expected, JobState next, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
|
||||
var sql = $"""
|
||||
UPDATE {Table}
|
||||
SET state = @next::job_state,
|
||||
heartbeat_at_utc = @heartbeat,
|
||||
started_at_utc = CASE
|
||||
WHEN @next = 'Running' THEN COALESCE(started_at_utc, @heartbeat)
|
||||
ELSE started_at_utc END,
|
||||
completed_at_utc = CASE
|
||||
WHEN @next IN ('Succeeded','Failed','Cancelled') THEN @heartbeat
|
||||
ELSE completed_at_utc END
|
||||
WHERE id = @id AND state = @expected::job_state
|
||||
""";
|
||||
|
||||
var affected = await ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", jobId);
|
||||
AddParameter(cmd, "expected", expected.ToString());
|
||||
AddParameter(cmd, "next", next.ToString());
|
||||
AddParameter(cmd, "heartbeat", now);
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return affected == 1;
|
||||
}
|
||||
|
||||
public Task<JobDocument?> GetAsync(string jobId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
FROM {Table}
|
||||
WHERE id = @id
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "id", jobId),
|
||||
MapJob,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<List<JobDocument>> ListStaleAsync(TimeSpan heartbeatThreshold, CancellationToken cancellationToken)
|
||||
{
|
||||
if (heartbeatThreshold <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(heartbeatThreshold));
|
||||
}
|
||||
|
||||
var cutoff = _timeProvider.GetUtcNow().UtcDateTime - heartbeatThreshold;
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
|
||||
FROM {Table}
|
||||
WHERE state = 'Running'::job_state
|
||||
AND heartbeat_at_utc < @cutoff
|
||||
ORDER BY heartbeat_at_utc
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "cutoff", cutoff),
|
||||
MapJob,
|
||||
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
|
||||
}
|
||||
|
||||
private static JobDocument MapJob(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
Kind = reader.GetString(reader.GetOrdinal("kind")),
|
||||
State = Enum.TryParse<JobState>(reader.GetString(reader.GetOrdinal("state")), true, out var state)
|
||||
? state
|
||||
: JobState.Pending,
|
||||
ArgumentsJson = reader.GetString(reader.GetOrdinal("args")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
StartedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("started_at_utc"))?.UtcDateTime,
|
||||
CompletedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("completed_at_utc"))?.UtcDateTime,
|
||||
HeartbeatAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("heartbeat_at_utc"))?.UtcDateTime,
|
||||
Error = GetNullableString(reader, reader.GetOrdinal("error"))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,79 +1,79 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class LayerRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.layers";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public LayerRepository(ScannerDataSource dataSource, ILogger<LayerRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.LastSeenAtUtc = now;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc)
|
||||
VALUES (@layer_digest, @media_type, @size_bytes, @created_at_utc, @last_seen_at_utc)
|
||||
ON CONFLICT (layer_digest) DO UPDATE SET
|
||||
media_type = EXCLUDED.media_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
last_seen_at_utc = EXCLUDED.last_seen_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "layer_digest", document.LayerDigest);
|
||||
AddParameter(cmd, "media_type", document.MediaType);
|
||||
AddParameter(cmd, "size_bytes", document.SizeBytes);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<LayerDocument?> GetAsync(string layerDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
|
||||
var sql = $"""
|
||||
SELECT layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc
|
||||
FROM {Table}
|
||||
WHERE layer_digest = @layer_digest
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "layer_digest", layerDigest),
|
||||
MapLayer,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static LayerDocument MapLayer(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
LayerDigest = reader.GetString(reader.GetOrdinal("layer_digest")),
|
||||
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
|
||||
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
|
||||
};
|
||||
}
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class LayerRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.layers";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public LayerRepository(ScannerDataSource dataSource, ILogger<LayerRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.LastSeenAtUtc = now;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc)
|
||||
VALUES (@layer_digest, @media_type, @size_bytes, @created_at_utc, @last_seen_at_utc)
|
||||
ON CONFLICT (layer_digest) DO UPDATE SET
|
||||
media_type = EXCLUDED.media_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
last_seen_at_utc = EXCLUDED.last_seen_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "layer_digest", document.LayerDigest);
|
||||
AddParameter(cmd, "media_type", document.MediaType);
|
||||
AddParameter(cmd, "size_bytes", document.SizeBytes);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<LayerDocument?> GetAsync(string layerDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
|
||||
var sql = $"""
|
||||
SELECT layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc
|
||||
FROM {Table}
|
||||
WHERE layer_digest = @layer_digest
|
||||
""";
|
||||
|
||||
return QuerySingleOrDefaultAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "layer_digest", layerDigest),
|
||||
MapLayer,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static LayerDocument MapLayer(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
LayerDigest = reader.GetString(reader.GetOrdinal("layer_digest")),
|
||||
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
|
||||
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,77 +1,77 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class LifecycleRuleRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.lifecycle_rules";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public LifecycleRuleRepository(ScannerDataSource dataSource, ILogger<LifecycleRuleRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (id, artifact_id, class, expires_at_utc, created_at_utc)
|
||||
VALUES (@id, @artifact_id, @class, @expires_at_utc, @created_at_utc)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
artifact_id = EXCLUDED.artifact_id,
|
||||
class = EXCLUDED.class,
|
||||
expires_at_utc = EXCLUDED.expires_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "artifact_id", document.ArtifactId);
|
||||
AddParameter(cmd, "class", document.Class);
|
||||
AddParameter(cmd, "expires_at_utc", document.ExpiresAtUtc);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<List<LifecycleRuleDocument>> ListExpiredAsync(DateTime utcNow, CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
SELECT id, artifact_id, class, expires_at_utc, created_at_utc
|
||||
FROM {Table}
|
||||
WHERE expires_at_utc IS NOT NULL AND expires_at_utc < @now
|
||||
ORDER BY expires_at_utc
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "now", utcNow),
|
||||
MapRule,
|
||||
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
|
||||
}
|
||||
|
||||
private static LifecycleRuleDocument MapRule(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
|
||||
Class = reader.GetString(reader.GetOrdinal("class")),
|
||||
ExpiresAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("expires_at_utc"))?.UtcDateTime,
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
};
|
||||
}
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class LifecycleRuleRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.lifecycle_rules";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public LifecycleRuleRepository(ScannerDataSource dataSource, ILogger<LifecycleRuleRepository> logger, TimeProvider? timeProvider = null)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (id, artifact_id, class, expires_at_utc, created_at_utc)
|
||||
VALUES (@id, @artifact_id, @class, @expires_at_utc, @created_at_utc)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
artifact_id = EXCLUDED.artifact_id,
|
||||
class = EXCLUDED.class,
|
||||
expires_at_utc = EXCLUDED.expires_at_utc
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "artifact_id", document.ArtifactId);
|
||||
AddParameter(cmd, "class", document.Class);
|
||||
AddParameter(cmd, "expires_at_utc", document.ExpiresAtUtc);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<List<LifecycleRuleDocument>> ListExpiredAsync(DateTime utcNow, CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
SELECT id, artifact_id, class, expires_at_utc, created_at_utc
|
||||
FROM {Table}
|
||||
WHERE expires_at_utc IS NOT NULL AND expires_at_utc < @now
|
||||
ORDER BY expires_at_utc
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddParameter(cmd, "now", utcNow),
|
||||
MapRule,
|
||||
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
|
||||
}
|
||||
|
||||
private static LifecycleRuleDocument MapRule(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
|
||||
Class = reader.GetString(reader.GetOrdinal("class")),
|
||||
ExpiresAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("expires_at_utc"))?.UtcDateTime,
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,80 +1,80 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class LinkRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.links";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
public LinkRepository(ScannerDataSource dataSource, ILogger<LinkRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (id, from_type, from_digest, artifact_id, created_at_utc)
|
||||
VALUES (@id, @from_type, @from_digest, @artifact_id, @created_at_utc)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
from_type = EXCLUDED.from_type,
|
||||
from_digest = EXCLUDED.from_digest,
|
||||
artifact_id = EXCLUDED.artifact_id
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "from_type", document.FromType.ToString());
|
||||
AddParameter(cmd, "from_digest", document.FromDigest);
|
||||
AddParameter(cmd, "artifact_id", document.ArtifactId);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<List<LinkDocument>> ListBySourceAsync(LinkSourceType type, string digest, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, from_type, from_digest, artifact_id, created_at_utc
|
||||
FROM {Table}
|
||||
WHERE from_type = @from_type AND from_digest = @from_digest
|
||||
ORDER BY created_at_utc DESC, id
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "from_type", type.ToString());
|
||||
AddParameter(cmd, "from_digest", digest);
|
||||
},
|
||||
MapLink,
|
||||
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
|
||||
}
|
||||
|
||||
private static LinkDocument MapLink(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
FromType = Enum.TryParse<LinkSourceType>(reader.GetString(reader.GetOrdinal("from_type")), true, out var parsed)
|
||||
? parsed
|
||||
: LinkSourceType.Image,
|
||||
FromDigest = reader.GetString(reader.GetOrdinal("from_digest")),
|
||||
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
};
|
||||
}
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class LinkRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.links";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
|
||||
public LinkRepository(ScannerDataSource dataSource, ILogger<LinkRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (id, from_type, from_digest, artifact_id, created_at_utc)
|
||||
VALUES (@id, @from_type, @from_digest, @artifact_id, @created_at_utc)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
from_type = EXCLUDED.from_type,
|
||||
from_digest = EXCLUDED.from_digest,
|
||||
artifact_id = EXCLUDED.artifact_id
|
||||
""";
|
||||
|
||||
return ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", document.Id);
|
||||
AddParameter(cmd, "from_type", document.FromType.ToString());
|
||||
AddParameter(cmd, "from_digest", document.FromDigest);
|
||||
AddParameter(cmd, "artifact_id", document.ArtifactId);
|
||||
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task<List<LinkDocument>> ListBySourceAsync(LinkSourceType type, string digest, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
|
||||
|
||||
var sql = $"""
|
||||
SELECT id, from_type, from_digest, artifact_id, created_at_utc
|
||||
FROM {Table}
|
||||
WHERE from_type = @from_type AND from_digest = @from_digest
|
||||
ORDER BY created_at_utc DESC, id
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "from_type", type.ToString());
|
||||
AddParameter(cmd, "from_digest", digest);
|
||||
},
|
||||
MapLink,
|
||||
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
|
||||
}
|
||||
|
||||
private static LinkDocument MapLink(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
FromType = Enum.TryParse<LinkSourceType>(reader.GetString(reader.GetOrdinal("from_type")), true, out var parsed)
|
||||
? parsed
|
||||
: LinkSourceType.Image,
|
||||
FromDigest = reader.GetString(reader.GetOrdinal("from_digest")),
|
||||
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
|
||||
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,247 +1,247 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// Repository responsible for persisting runtime events.
|
||||
/// </summary>
|
||||
public sealed class RuntimeEventRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.runtime_events";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public RuntimeEventRepository(ScannerDataSource dataSource, ILogger<RuntimeEventRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<RuntimeEventInsertResult> InsertAsync(
|
||||
IReadOnlyCollection<RuntimeEventDocument> documents,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(documents);
|
||||
if (documents.Count == 0)
|
||||
{
|
||||
return RuntimeEventInsertResult.Empty;
|
||||
}
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
|
||||
platform, namespace, pod, container, container_id, image_ref, image_digest,
|
||||
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
|
||||
)
|
||||
VALUES (
|
||||
@id, @event_id, @schema_version, @tenant, @node, @kind, @when, @received_at, @expires_at,
|
||||
@platform, @namespace, @pod, @container, @container_id, @image_ref, @image_digest,
|
||||
@engine, @engine_version, @baseline_digest, @image_signed, @sbom_referrer, @build_id, @payload::jsonb
|
||||
)
|
||||
ON CONFLICT (event_id) DO NOTHING
|
||||
""";
|
||||
|
||||
var inserted = 0;
|
||||
var duplicates = 0;
|
||||
|
||||
foreach (var document in documents)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var id = string.IsNullOrWhiteSpace(document.Id) ? Guid.NewGuid().ToString("N") : document.Id;
|
||||
|
||||
var rows = await ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", id);
|
||||
AddParameter(cmd, "event_id", document.EventId);
|
||||
AddParameter(cmd, "schema_version", document.SchemaVersion);
|
||||
AddParameter(cmd, "tenant", document.Tenant);
|
||||
AddParameter(cmd, "node", document.Node);
|
||||
AddParameter(cmd, "kind", document.Kind);
|
||||
AddParameter(cmd, "when", document.When);
|
||||
AddParameter(cmd, "received_at", document.ReceivedAt);
|
||||
AddParameter(cmd, "expires_at", document.ExpiresAt);
|
||||
AddParameter(cmd, "platform", document.Platform);
|
||||
AddParameter(cmd, "namespace", document.Namespace);
|
||||
AddParameter(cmd, "pod", document.Pod);
|
||||
AddParameter(cmd, "container", document.Container);
|
||||
AddParameter(cmd, "container_id", document.ContainerId);
|
||||
AddParameter(cmd, "image_ref", document.ImageRef);
|
||||
AddParameter(cmd, "image_digest", document.ImageDigest);
|
||||
AddParameter(cmd, "engine", document.Engine);
|
||||
AddParameter(cmd, "engine_version", document.EngineVersion);
|
||||
AddParameter(cmd, "baseline_digest", document.BaselineDigest);
|
||||
AddParameter(cmd, "image_signed", document.ImageSigned);
|
||||
AddParameter(cmd, "sbom_referrer", document.SbomReferrer);
|
||||
AddParameter(cmd, "build_id", document.BuildId);
|
||||
AddJsonbParameter(cmd, "payload", document.PayloadJson);
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (rows > 0)
|
||||
{
|
||||
inserted++;
|
||||
}
|
||||
else
|
||||
{
|
||||
duplicates++;
|
||||
}
|
||||
}
|
||||
|
||||
return new RuntimeEventInsertResult(inserted, duplicates);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyDictionary<string, RuntimeBuildIdObservation>> GetRecentBuildIdsAsync(
|
||||
IReadOnlyCollection<string> imageDigests,
|
||||
int maxPerImage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(imageDigests);
|
||||
if (imageDigests.Count == 0 || maxPerImage <= 0)
|
||||
{
|
||||
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
var normalized = imageDigests
|
||||
.Where(digest => !string.IsNullOrWhiteSpace(digest))
|
||||
.Select(digest => digest.Trim().ToLowerInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
if (normalized.Length == 0)
|
||||
{
|
||||
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
var sql = $"""
|
||||
SELECT image_digest, build_id, "when"
|
||||
FROM {Table}
|
||||
WHERE image_digest = ANY(@digests)
|
||||
AND build_id IS NOT NULL
|
||||
AND build_id <> ''
|
||||
ORDER BY image_digest, "when" DESC
|
||||
""";
|
||||
|
||||
var rows = await QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddTextArrayParameter(cmd, "digests", normalized),
|
||||
reader => new
|
||||
{
|
||||
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
|
||||
BuildId = reader.GetString(reader.GetOrdinal("build_id")),
|
||||
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when"))
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var result = new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
|
||||
foreach (var group in rows.GroupBy(r => r.ImageDigest, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var buildIds = group
|
||||
.Select(r => r.BuildId)
|
||||
.Where(id => !string.IsNullOrWhiteSpace(id))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(maxPerImage)
|
||||
.Select(id => id.Trim().ToLowerInvariant())
|
||||
.ToArray();
|
||||
|
||||
if (buildIds.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var observedAt = group.Select(r => r.When).FirstOrDefault();
|
||||
result[group.Key.ToLowerInvariant()] = new RuntimeBuildIdObservation(group.Key, buildIds, observedAt);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public Task<int> CountAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
SELECT COUNT(*) FROM {Table}
|
||||
""";
|
||||
|
||||
return ExecuteScalarAsync<int>(
|
||||
Tenant,
|
||||
sql,
|
||||
null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task TruncateAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
TRUNCATE TABLE {Table} RESTART IDENTITY CASCADE
|
||||
""";
|
||||
|
||||
return ExecuteAsync(Tenant, sql, null, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<RuntimeEventDocument>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
SELECT id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
|
||||
platform, namespace, pod, container, container_id, image_ref, image_digest,
|
||||
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
|
||||
FROM {Table}
|
||||
ORDER BY received_at
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
null,
|
||||
MapRuntimeEvent,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static RuntimeEventDocument MapRuntimeEvent(NpgsqlDataReader reader)
|
||||
{
|
||||
var payloadOrdinal = reader.GetOrdinal("payload");
|
||||
return new RuntimeEventDocument
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
EventId = reader.GetString(reader.GetOrdinal("event_id")),
|
||||
SchemaVersion = reader.GetString(reader.GetOrdinal("schema_version")),
|
||||
Tenant = reader.GetString(reader.GetOrdinal("tenant")),
|
||||
Node = reader.GetString(reader.GetOrdinal("node")),
|
||||
Kind = reader.GetString(reader.GetOrdinal("kind")),
|
||||
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when")),
|
||||
ReceivedAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("received_at")),
|
||||
ExpiresAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("expires_at")),
|
||||
Platform = GetNullableString(reader, reader.GetOrdinal("platform")),
|
||||
Namespace = GetNullableString(reader, reader.GetOrdinal("namespace")),
|
||||
Pod = GetNullableString(reader, reader.GetOrdinal("pod")),
|
||||
Container = GetNullableString(reader, reader.GetOrdinal("container")),
|
||||
ContainerId = GetNullableString(reader, reader.GetOrdinal("container_id")),
|
||||
ImageRef = GetNullableString(reader, reader.GetOrdinal("image_ref")),
|
||||
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
|
||||
Engine = GetNullableString(reader, reader.GetOrdinal("engine")),
|
||||
EngineVersion = GetNullableString(reader, reader.GetOrdinal("engine_version")),
|
||||
BaselineDigest = GetNullableString(reader, reader.GetOrdinal("baseline_digest")),
|
||||
ImageSigned = GetNullableBoolean(reader, reader.GetOrdinal("image_signed")),
|
||||
SbomReferrer = GetNullableString(reader, reader.GetOrdinal("sbom_referrer")),
|
||||
BuildId = GetNullableString(reader, reader.GetOrdinal("build_id")),
|
||||
PayloadJson = reader.IsDBNull(payloadOrdinal) ? "{}" : reader.GetString(payloadOrdinal)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public readonly record struct RuntimeEventInsertResult(int InsertedCount, int DuplicateCount)
|
||||
{
|
||||
public static RuntimeEventInsertResult Empty => new(0, 0);
|
||||
}
|
||||
|
||||
public sealed record RuntimeBuildIdObservation(
|
||||
string ImageDigest,
|
||||
IReadOnlyList<string> BuildIds,
|
||||
DateTime ObservedAtUtc);
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// Repository responsible for persisting runtime events.
|
||||
/// </summary>
|
||||
public sealed class RuntimeEventRepository : RepositoryBase<ScannerDataSource>
|
||||
{
|
||||
private const string Tenant = "";
|
||||
private string Table => $"{SchemaName}.runtime_events";
|
||||
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public RuntimeEventRepository(ScannerDataSource dataSource, ILogger<RuntimeEventRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<RuntimeEventInsertResult> InsertAsync(
|
||||
IReadOnlyCollection<RuntimeEventDocument> documents,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(documents);
|
||||
if (documents.Count == 0)
|
||||
{
|
||||
return RuntimeEventInsertResult.Empty;
|
||||
}
|
||||
|
||||
var sql = $"""
|
||||
INSERT INTO {Table} (
|
||||
id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
|
||||
platform, namespace, pod, container, container_id, image_ref, image_digest,
|
||||
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
|
||||
)
|
||||
VALUES (
|
||||
@id, @event_id, @schema_version, @tenant, @node, @kind, @when, @received_at, @expires_at,
|
||||
@platform, @namespace, @pod, @container, @container_id, @image_ref, @image_digest,
|
||||
@engine, @engine_version, @baseline_digest, @image_signed, @sbom_referrer, @build_id, @payload::jsonb
|
||||
)
|
||||
ON CONFLICT (event_id) DO NOTHING
|
||||
""";
|
||||
|
||||
var inserted = 0;
|
||||
var duplicates = 0;
|
||||
|
||||
foreach (var document in documents)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var id = string.IsNullOrWhiteSpace(document.Id) ? Guid.NewGuid().ToString("N") : document.Id;
|
||||
|
||||
var rows = await ExecuteAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd =>
|
||||
{
|
||||
AddParameter(cmd, "id", id);
|
||||
AddParameter(cmd, "event_id", document.EventId);
|
||||
AddParameter(cmd, "schema_version", document.SchemaVersion);
|
||||
AddParameter(cmd, "tenant", document.Tenant);
|
||||
AddParameter(cmd, "node", document.Node);
|
||||
AddParameter(cmd, "kind", document.Kind);
|
||||
AddParameter(cmd, "when", document.When);
|
||||
AddParameter(cmd, "received_at", document.ReceivedAt);
|
||||
AddParameter(cmd, "expires_at", document.ExpiresAt);
|
||||
AddParameter(cmd, "platform", document.Platform);
|
||||
AddParameter(cmd, "namespace", document.Namespace);
|
||||
AddParameter(cmd, "pod", document.Pod);
|
||||
AddParameter(cmd, "container", document.Container);
|
||||
AddParameter(cmd, "container_id", document.ContainerId);
|
||||
AddParameter(cmd, "image_ref", document.ImageRef);
|
||||
AddParameter(cmd, "image_digest", document.ImageDigest);
|
||||
AddParameter(cmd, "engine", document.Engine);
|
||||
AddParameter(cmd, "engine_version", document.EngineVersion);
|
||||
AddParameter(cmd, "baseline_digest", document.BaselineDigest);
|
||||
AddParameter(cmd, "image_signed", document.ImageSigned);
|
||||
AddParameter(cmd, "sbom_referrer", document.SbomReferrer);
|
||||
AddParameter(cmd, "build_id", document.BuildId);
|
||||
AddJsonbParameter(cmd, "payload", document.PayloadJson);
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (rows > 0)
|
||||
{
|
||||
inserted++;
|
||||
}
|
||||
else
|
||||
{
|
||||
duplicates++;
|
||||
}
|
||||
}
|
||||
|
||||
return new RuntimeEventInsertResult(inserted, duplicates);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyDictionary<string, RuntimeBuildIdObservation>> GetRecentBuildIdsAsync(
|
||||
IReadOnlyCollection<string> imageDigests,
|
||||
int maxPerImage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(imageDigests);
|
||||
if (imageDigests.Count == 0 || maxPerImage <= 0)
|
||||
{
|
||||
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
var normalized = imageDigests
|
||||
.Where(digest => !string.IsNullOrWhiteSpace(digest))
|
||||
.Select(digest => digest.Trim().ToLowerInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
if (normalized.Length == 0)
|
||||
{
|
||||
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
var sql = $"""
|
||||
SELECT image_digest, build_id, "when"
|
||||
FROM {Table}
|
||||
WHERE image_digest = ANY(@digests)
|
||||
AND build_id IS NOT NULL
|
||||
AND build_id <> ''
|
||||
ORDER BY image_digest, "when" DESC
|
||||
""";
|
||||
|
||||
var rows = await QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
cmd => AddTextArrayParameter(cmd, "digests", normalized),
|
||||
reader => new
|
||||
{
|
||||
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
|
||||
BuildId = reader.GetString(reader.GetOrdinal("build_id")),
|
||||
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when"))
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var result = new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
|
||||
foreach (var group in rows.GroupBy(r => r.ImageDigest, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var buildIds = group
|
||||
.Select(r => r.BuildId)
|
||||
.Where(id => !string.IsNullOrWhiteSpace(id))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(maxPerImage)
|
||||
.Select(id => id.Trim().ToLowerInvariant())
|
||||
.ToArray();
|
||||
|
||||
if (buildIds.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var observedAt = group.Select(r => r.When).FirstOrDefault();
|
||||
result[group.Key.ToLowerInvariant()] = new RuntimeBuildIdObservation(group.Key, buildIds, observedAt);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public Task<int> CountAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
SELECT COUNT(*) FROM {Table}
|
||||
""";
|
||||
|
||||
return ExecuteScalarAsync<int>(
|
||||
Tenant,
|
||||
sql,
|
||||
null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task TruncateAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
TRUNCATE TABLE {Table} RESTART IDENTITY CASCADE
|
||||
""";
|
||||
|
||||
return ExecuteAsync(Tenant, sql, null, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<RuntimeEventDocument>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sql = $"""
|
||||
SELECT id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
|
||||
platform, namespace, pod, container, container_id, image_ref, image_digest,
|
||||
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
|
||||
FROM {Table}
|
||||
ORDER BY received_at
|
||||
""";
|
||||
|
||||
return QueryAsync(
|
||||
Tenant,
|
||||
sql,
|
||||
null,
|
||||
MapRuntimeEvent,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static RuntimeEventDocument MapRuntimeEvent(NpgsqlDataReader reader)
|
||||
{
|
||||
var payloadOrdinal = reader.GetOrdinal("payload");
|
||||
return new RuntimeEventDocument
|
||||
{
|
||||
Id = reader.GetString(reader.GetOrdinal("id")),
|
||||
EventId = reader.GetString(reader.GetOrdinal("event_id")),
|
||||
SchemaVersion = reader.GetString(reader.GetOrdinal("schema_version")),
|
||||
Tenant = reader.GetString(reader.GetOrdinal("tenant")),
|
||||
Node = reader.GetString(reader.GetOrdinal("node")),
|
||||
Kind = reader.GetString(reader.GetOrdinal("kind")),
|
||||
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when")),
|
||||
ReceivedAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("received_at")),
|
||||
ExpiresAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("expires_at")),
|
||||
Platform = GetNullableString(reader, reader.GetOrdinal("platform")),
|
||||
Namespace = GetNullableString(reader, reader.GetOrdinal("namespace")),
|
||||
Pod = GetNullableString(reader, reader.GetOrdinal("pod")),
|
||||
Container = GetNullableString(reader, reader.GetOrdinal("container")),
|
||||
ContainerId = GetNullableString(reader, reader.GetOrdinal("container_id")),
|
||||
ImageRef = GetNullableString(reader, reader.GetOrdinal("image_ref")),
|
||||
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
|
||||
Engine = GetNullableString(reader, reader.GetOrdinal("engine")),
|
||||
EngineVersion = GetNullableString(reader, reader.GetOrdinal("engine_version")),
|
||||
BaselineDigest = GetNullableString(reader, reader.GetOrdinal("baseline_digest")),
|
||||
ImageSigned = GetNullableBoolean(reader, reader.GetOrdinal("image_signed")),
|
||||
SbomReferrer = GetNullableString(reader, reader.GetOrdinal("sbom_referrer")),
|
||||
BuildId = GetNullableString(reader, reader.GetOrdinal("build_id")),
|
||||
PayloadJson = reader.IsDBNull(payloadOrdinal) ? "{}" : reader.GetString(payloadOrdinal)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public readonly record struct RuntimeEventInsertResult(int InsertedCount, int DuplicateCount)
|
||||
{
|
||||
public static RuntimeEventInsertResult Empty => new(0, 0);
|
||||
}
|
||||
|
||||
public sealed record RuntimeBuildIdObservation(
|
||||
string ImageDigest,
|
||||
IReadOnlyList<string> BuildIds,
|
||||
DateTime ObservedAtUtc);
|
||||
|
||||
@@ -1,144 +1,144 @@
|
||||
using System.Buffers;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Services;
|
||||
|
||||
public sealed class ArtifactStorageService
|
||||
{
|
||||
private readonly ArtifactRepository _artifactRepository;
|
||||
private readonly LifecycleRuleRepository _lifecycleRuleRepository;
|
||||
private readonly IArtifactObjectStore _objectStore;
|
||||
private readonly ScannerStorageOptions _options;
|
||||
private readonly ILogger<ArtifactStorageService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ArtifactStorageService(
|
||||
ArtifactRepository artifactRepository,
|
||||
LifecycleRuleRepository lifecycleRuleRepository,
|
||||
IArtifactObjectStore objectStore,
|
||||
IOptions<ScannerStorageOptions> options,
|
||||
ILogger<ArtifactStorageService> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
|
||||
_lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository));
|
||||
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task<ArtifactDocument> StoreArtifactAsync(
|
||||
ArtifactDocumentType type,
|
||||
ArtifactDocumentFormat format,
|
||||
string mediaType,
|
||||
Stream content,
|
||||
bool immutable,
|
||||
string ttlClass,
|
||||
DateTime? expiresAtUtc,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
|
||||
|
||||
var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var normalizedDigest = $"sha256:{digestHex}";
|
||||
using System.Buffers;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Services;
|
||||
|
||||
public sealed class ArtifactStorageService
|
||||
{
|
||||
private readonly ArtifactRepository _artifactRepository;
|
||||
private readonly LifecycleRuleRepository _lifecycleRuleRepository;
|
||||
private readonly IArtifactObjectStore _objectStore;
|
||||
private readonly ScannerStorageOptions _options;
|
||||
private readonly ILogger<ArtifactStorageService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ArtifactStorageService(
|
||||
ArtifactRepository artifactRepository,
|
||||
LifecycleRuleRepository lifecycleRuleRepository,
|
||||
IArtifactObjectStore objectStore,
|
||||
IOptions<ScannerStorageOptions> options,
|
||||
ILogger<ArtifactStorageService> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
|
||||
_lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository));
|
||||
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task<ArtifactDocument> StoreArtifactAsync(
|
||||
ArtifactDocumentType type,
|
||||
ArtifactDocumentFormat format,
|
||||
string mediaType,
|
||||
Stream content,
|
||||
bool immutable,
|
||||
string ttlClass,
|
||||
DateTime? expiresAtUtc,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
|
||||
|
||||
var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var normalizedDigest = $"sha256:{digestHex}";
|
||||
var artifactId = CatalogIdFactory.CreateArtifactId(type, normalizedDigest);
|
||||
var key = ArtifactObjectKeyBuilder.Build(
|
||||
type,
|
||||
format,
|
||||
normalizedDigest,
|
||||
_options.ObjectStore.RootPrefix);
|
||||
var descriptor = new ArtifactObjectDescriptor(
|
||||
_options.ObjectStore.BucketName,
|
||||
key,
|
||||
immutable,
|
||||
_options.ObjectStore.ComplianceRetention);
|
||||
|
||||
buffer.Position = 0;
|
||||
await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.DualWrite.Enabled)
|
||||
{
|
||||
buffer.Position = 0;
|
||||
var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! };
|
||||
await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
var document = new ArtifactDocument
|
||||
{
|
||||
Id = artifactId,
|
||||
Type = type,
|
||||
Format = format,
|
||||
MediaType = mediaType,
|
||||
BytesSha256 = normalizedDigest,
|
||||
SizeBytes = size,
|
||||
Immutable = immutable,
|
||||
RefCount = 1,
|
||||
CreatedAtUtc = now,
|
||||
UpdatedAtUtc = now,
|
||||
TtlClass = ttlClass,
|
||||
};
|
||||
|
||||
await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (expiresAtUtc.HasValue)
|
||||
{
|
||||
var lifecycle = new LifecycleRuleDocument
|
||||
{
|
||||
Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass),
|
||||
ArtifactId = document.Id,
|
||||
Class = ttlClass,
|
||||
ExpiresAtUtc = expiresAtUtc,
|
||||
CreatedAtUtc = now,
|
||||
};
|
||||
|
||||
await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest);
|
||||
return document;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await buffer.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
var bufferStream = new MemoryStream();
|
||||
var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var rented = ArrayPool<byte>.Shared.Rent(81920);
|
||||
long total = 0;
|
||||
|
||||
try
|
||||
{
|
||||
int read;
|
||||
while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hasher.AppendData(rented, 0, read);
|
||||
await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
|
||||
total += read;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(rented);
|
||||
}
|
||||
|
||||
bufferStream.Position = 0;
|
||||
var digest = hasher.GetCurrentHash();
|
||||
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
|
||||
return (bufferStream, total, digestHex);
|
||||
}
|
||||
|
||||
var descriptor = new ArtifactObjectDescriptor(
|
||||
_options.ObjectStore.BucketName,
|
||||
key,
|
||||
immutable,
|
||||
_options.ObjectStore.ComplianceRetention);
|
||||
|
||||
buffer.Position = 0;
|
||||
await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.DualWrite.Enabled)
|
||||
{
|
||||
buffer.Position = 0;
|
||||
var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! };
|
||||
await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
var document = new ArtifactDocument
|
||||
{
|
||||
Id = artifactId,
|
||||
Type = type,
|
||||
Format = format,
|
||||
MediaType = mediaType,
|
||||
BytesSha256 = normalizedDigest,
|
||||
SizeBytes = size,
|
||||
Immutable = immutable,
|
||||
RefCount = 1,
|
||||
CreatedAtUtc = now,
|
||||
UpdatedAtUtc = now,
|
||||
TtlClass = ttlClass,
|
||||
};
|
||||
|
||||
await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (expiresAtUtc.HasValue)
|
||||
{
|
||||
var lifecycle = new LifecycleRuleDocument
|
||||
{
|
||||
Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass),
|
||||
ArtifactId = document.Id,
|
||||
Class = ttlClass,
|
||||
ExpiresAtUtc = expiresAtUtc,
|
||||
CreatedAtUtc = now,
|
||||
};
|
||||
|
||||
await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest);
|
||||
return document;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await buffer.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
var bufferStream = new MemoryStream();
|
||||
var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var rented = ArrayPool<byte>.Shared.Rent(81920);
|
||||
long total = 0;
|
||||
|
||||
try
|
||||
{
|
||||
int read;
|
||||
while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hasher.AppendData(rented, 0, read);
|
||||
await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
|
||||
total += read;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(rented);
|
||||
}
|
||||
|
||||
bufferStream.Position = 0;
|
||||
var digest = hasher.GetCurrentHash();
|
||||
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
|
||||
return (bufferStream, total, digestHex);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user