stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

274
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Services/ArtifactStorageService.cs
View File

@@ -1,144 +1,144 @@
using System.Buffers;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Services;
public sealed class ArtifactStorageService
{
private readonly ArtifactRepository _artifactRepository;
private readonly LifecycleRuleRepository _lifecycleRuleRepository;
private readonly IArtifactObjectStore _objectStore;
private readonly ScannerStorageOptions _options;
private readonly ILogger<ArtifactStorageService> _logger;
private readonly TimeProvider _timeProvider;
public ArtifactStorageService(
ArtifactRepository artifactRepository,
LifecycleRuleRepository lifecycleRuleRepository,
IArtifactObjectStore objectStore,
IOptions<ScannerStorageOptions> options,
ILogger<ArtifactStorageService> logger,
TimeProvider? timeProvider = null)
{
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
_lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<ArtifactDocument> StoreArtifactAsync(
ArtifactDocumentType type,
ArtifactDocumentFormat format,
string mediaType,
Stream content,
bool immutable,
string ttlClass,
DateTime? expiresAtUtc,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(content);
ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false);
try
{
var normalizedDigest = $"sha256:{digestHex}";
using System.Buffers;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Services;
public sealed class ArtifactStorageService
{
private readonly ArtifactRepository _artifactRepository;
private readonly LifecycleRuleRepository _lifecycleRuleRepository;
private readonly IArtifactObjectStore _objectStore;
private readonly ScannerStorageOptions _options;
private readonly ILogger<ArtifactStorageService> _logger;
private readonly TimeProvider _timeProvider;
public ArtifactStorageService(
ArtifactRepository artifactRepository,
LifecycleRuleRepository lifecycleRuleRepository,
IArtifactObjectStore objectStore,
IOptions<ScannerStorageOptions> options,
ILogger<ArtifactStorageService> logger,
TimeProvider? timeProvider = null)
{
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
_lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<ArtifactDocument> StoreArtifactAsync(
ArtifactDocumentType type,
ArtifactDocumentFormat format,
string mediaType,
Stream content,
bool immutable,
string ttlClass,
DateTime? expiresAtUtc,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(content);
ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false);
try
{
var normalizedDigest = $"sha256:{digestHex}";
var artifactId = CatalogIdFactory.CreateArtifactId(type, normalizedDigest);
var key = ArtifactObjectKeyBuilder.Build(
type,
format,
normalizedDigest,
_options.ObjectStore.RootPrefix);
var descriptor = new ArtifactObjectDescriptor(
_options.ObjectStore.BucketName,
key,
immutable,
_options.ObjectStore.ComplianceRetention);
buffer.Position = 0;
await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false);
if (_options.DualWrite.Enabled)
{
buffer.Position = 0;
var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! };
await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false);
}
var now = _timeProvider.GetUtcNow().UtcDateTime;
var document = new ArtifactDocument
{
Id = artifactId,
Type = type,
Format = format,
MediaType = mediaType,
BytesSha256 = normalizedDigest,
SizeBytes = size,
Immutable = immutable,
RefCount = 1,
CreatedAtUtc = now,
UpdatedAtUtc = now,
TtlClass = ttlClass,
};
await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
if (expiresAtUtc.HasValue)
{
var lifecycle = new LifecycleRuleDocument
{
Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass),
ArtifactId = document.Id,
Class = ttlClass,
ExpiresAtUtc = expiresAtUtc,
CreatedAtUtc = now,
};
await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest);
return document;
}
finally
{
await buffer.DisposeAsync().ConfigureAwait(false);
}
}
private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken)
{
var bufferStream = new MemoryStream();
var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var rented = ArrayPool<byte>.Shared.Rent(81920);
long total = 0;
try
{
int read;
while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
hasher.AppendData(rented, 0, read);
await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
total += read;
}
}
finally
{
ArrayPool<byte>.Shared.Return(rented);
}
bufferStream.Position = 0;
var digest = hasher.GetCurrentHash();
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
return (bufferStream, total, digestHex);
}
var descriptor = new ArtifactObjectDescriptor(
_options.ObjectStore.BucketName,
key,
immutable,
_options.ObjectStore.ComplianceRetention);
buffer.Position = 0;
await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false);
if (_options.DualWrite.Enabled)
{
buffer.Position = 0;
var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! };
await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false);
}
var now = _timeProvider.GetUtcNow().UtcDateTime;
var document = new ArtifactDocument
{
Id = artifactId,
Type = type,
Format = format,
MediaType = mediaType,
BytesSha256 = normalizedDigest,
SizeBytes = size,
Immutable = immutable,
RefCount = 1,
CreatedAtUtc = now,
UpdatedAtUtc = now,
TtlClass = ttlClass,
};
await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
if (expiresAtUtc.HasValue)
{
var lifecycle = new LifecycleRuleDocument
{
Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass),
ArtifactId = document.Id,
Class = ttlClass,
ExpiresAtUtc = expiresAtUtc,
CreatedAtUtc = now,
};
await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest);
return document;
}
finally
{
await buffer.DisposeAsync().ConfigureAwait(false);
}
}
private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken)
{
var bufferStream = new MemoryStream();
var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var rented = ArrayPool<byte>.Shared.Rent(81920);
long total = 0;
try
{
int read;
while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
hasher.AppendData(rented, 0, read);
await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
total += read;
}
}
finally
{
ArrayPool<byte>.Shared.Return(rented);
}
bufferStream.Position = 0;
var digest = hasher.GetCurrentHash();
var digestHex = Convert.ToHexString(digest).ToLowerInvariant();
return (bufferStream, total, digestHex);
}
}