using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Signals.Options; using StellaOps.Signals.Storage.Models; using System; using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; namespace StellaOps.Signals.Storage; /// /// Stores callgraph artifacts in RustFS (S3-compatible content-addressable storage). /// internal sealed class RustFsCallgraphArtifactStore : ICallgraphArtifactStore { internal const string HttpClientName = "signals-storage-rustfs"; private const string DefaultFileName = "callgraph.json"; private const string ManifestFileName = "manifest.json"; private const string ImmutableHeader = "X-RustFS-Immutable"; private const string RetainSecondsHeader = "X-RustFS-Retain-Seconds"; private static readonly MediaTypeHeaderValue OctetStream = new("application/octet-stream"); /// /// Default retention for callgraph artifacts (90 days per CAS contract). /// private static readonly TimeSpan DefaultRetention = TimeSpan.FromDays(90); private readonly IHttpClientFactory _httpClientFactory; private readonly SignalsArtifactStorageOptions _storageOptions; private readonly ILogger _logger; public RustFsCallgraphArtifactStore( IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); ArgumentNullException.ThrowIfNull(options); _storageOptions = options.Value.Storage; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(request); ArgumentNullException.ThrowIfNull(content); var hash = NormalizeHash(request.Hash); if (string.IsNullOrWhiteSpace(hash)) { throw new InvalidOperationException("Callgraph artifact hash is required for CAS storage."); } var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? DefaultFileName : request.FileName); var objectKey = BuildObjectKey(hash, fileName); // Store the artifact await PutObjectAsync(objectKey, content, request.ContentType, cancellationToken).ConfigureAwait(false); // Store the manifest var manifestKey = BuildObjectKey(hash, ManifestFileName); if (request.ManifestContent != null) { request.ManifestContent.Position = 0; await PutObjectAsync(manifestKey, request.ManifestContent, "application/json", cancellationToken).ConfigureAwait(false); } else { // Create empty manifest placeholder using var emptyManifest = new MemoryStream(Encoding.UTF8.GetBytes("{}")); await PutObjectAsync(manifestKey, emptyManifest, "application/json", cancellationToken).ConfigureAwait(false); } var artifactLength = content.CanSeek ? content.Length : 0; _logger.LogInformation("Stored callgraph artifact {Hash}/{FileName} in RustFS bucket {Bucket}.", hash, fileName, _storageOptions.BucketName); return new StoredCallgraphArtifact( objectKey, artifactLength, hash, request.ContentType, $"cas://reachability/graphs/{hash}", manifestKey, $"cas://reachability/graphs/{hash}/manifest"); } public async Task GetAsync(string hash, string? fileName = null, CancellationToken cancellationToken = default) { var normalizedHash = NormalizeHash(hash); if (string.IsNullOrWhiteSpace(normalizedHash)) { return null; } var targetFileName = SanitizeFileName(string.IsNullOrWhiteSpace(fileName) ? DefaultFileName : fileName); var objectKey = BuildObjectKey(normalizedHash, targetFileName); var result = await GetObjectAsync(objectKey, cancellationToken).ConfigureAwait(false); if (result is null) { _logger.LogDebug("Callgraph artifact {Hash}/{FileName} not found in RustFS.", normalizedHash, targetFileName); } else { _logger.LogDebug("Retrieved callgraph artifact {Hash}/{FileName} from RustFS.", normalizedHash, targetFileName); } return result; } public async Task GetManifestAsync(string hash, CancellationToken cancellationToken = default) { var normalizedHash = NormalizeHash(hash); if (string.IsNullOrWhiteSpace(normalizedHash)) { return null; } var manifestKey = BuildObjectKey(normalizedHash, ManifestFileName); var result = await GetObjectAsync(manifestKey, cancellationToken).ConfigureAwait(false); if (result is null) { _logger.LogDebug("Callgraph manifest for {Hash} not found in RustFS.", normalizedHash); } else { _logger.LogDebug("Retrieved callgraph manifest for {Hash} from RustFS.", normalizedHash); } return result; } public async Task ExistsAsync(string hash, CancellationToken cancellationToken = default) { var normalizedHash = NormalizeHash(hash); if (string.IsNullOrWhiteSpace(normalizedHash)) { return false; } var objectKey = BuildObjectKey(normalizedHash, DefaultFileName); var exists = await HeadObjectAsync(objectKey, cancellationToken).ConfigureAwait(false); _logger.LogDebug("Callgraph artifact {Hash} exists={Exists} in RustFS.", normalizedHash, exists); return exists; } private string BuildObjectKey(string hash, string fileName) { var prefix = hash.Length >= 2 ? hash[..2] : hash; var rootPrefix = string.IsNullOrWhiteSpace(_storageOptions.RootPrefix) ? "callgraphs" : _storageOptions.RootPrefix; return $"{rootPrefix}/{prefix}/{hash}/{fileName}"; } private async Task PutObjectAsync(string objectKey, Stream content, string? contentType, CancellationToken cancellationToken) { var client = _httpClientFactory.CreateClient(HttpClientName); using var request = new HttpRequestMessage(HttpMethod.Put, BuildRequestUri(objectKey)) { Content = CreateHttpContent(content) }; request.Content.Headers.ContentType = string.IsNullOrWhiteSpace(contentType) ? OctetStream : new MediaTypeHeaderValue(contentType); ApplyHeaders(request); // Mark as immutable with 90-day retention per CAS contract request.Headers.TryAddWithoutValidation(ImmutableHeader, "true"); var retainSeconds = Math.Ceiling(DefaultRetention.TotalSeconds); request.Headers.TryAddWithoutValidation(RetainSecondsHeader, retainSeconds.ToString(CultureInfo.InvariantCulture)); using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false); throw new InvalidOperationException( $"RustFS upload for {_storageOptions.BucketName}/{objectKey} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}"); } _logger.LogDebug("Uploaded callgraph object {Bucket}/{Key} via RustFS.", _storageOptions.BucketName, objectKey); } private async Task GetObjectAsync(string objectKey, CancellationToken cancellationToken) { var client = _httpClientFactory.CreateClient(HttpClientName); using var request = new HttpRequestMessage(HttpMethod.Get, BuildRequestUri(objectKey)); ApplyHeaders(request); using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.NotFound) { return null; } if (!response.IsSuccessStatusCode) { var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false); throw new InvalidOperationException( $"RustFS download for {_storageOptions.BucketName}/{objectKey} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}"); } var buffer = new MemoryStream(); if (response.Content is not null) { await response.Content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); } buffer.Position = 0; return buffer; } private async Task HeadObjectAsync(string objectKey, CancellationToken cancellationToken) { var client = _httpClientFactory.CreateClient(HttpClientName); using var request = new HttpRequestMessage(HttpMethod.Head, BuildRequestUri(objectKey)); ApplyHeaders(request); using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); return response.StatusCode == HttpStatusCode.OK; } private Uri BuildRequestUri(string objectKey) { if (!Uri.TryCreate(_storageOptions.RustFs.BaseUrl, UriKind.Absolute, out var baseUri)) { throw new InvalidOperationException("RustFS baseUrl is invalid."); } var encodedBucket = Uri.EscapeDataString(_storageOptions.BucketName); var encodedKey = EncodeKey(objectKey); var relativePath = new StringBuilder() .Append("buckets/") .Append(encodedBucket) .Append("/objects/") .Append(encodedKey) .ToString(); return new Uri(baseUri, relativePath); } private static string EncodeKey(string key) { if (string.IsNullOrWhiteSpace(key)) { return string.Empty; } var segments = key.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); return string.Join('/', segments.Select(Uri.EscapeDataString)); } private void ApplyHeaders(HttpRequestMessage request) { var rustFsOptions = _storageOptions.RustFs; if (!string.IsNullOrWhiteSpace(rustFsOptions.ApiKeyHeader) && !string.IsNullOrWhiteSpace(rustFsOptions.ApiKey)) { request.Headers.TryAddWithoutValidation(rustFsOptions.ApiKeyHeader, rustFsOptions.ApiKey); } foreach (var header in _storageOptions.Headers) { request.Headers.TryAddWithoutValidation(header.Key, header.Value); } } private static HttpContent CreateHttpContent(Stream content) { if (content is MemoryStream memoryStream) { if (memoryStream.TryGetBuffer(out var segment)) { return new ByteArrayContent(segment.Array!, segment.Offset, segment.Count); } return new ByteArrayContent(memoryStream.ToArray()); } if (content.CanSeek) { var originalPosition = content.Position; try { content.Position = 0; using var duplicate = new MemoryStream(); content.CopyTo(duplicate); return new ByteArrayContent(duplicate.ToArray()); } finally { content.Position = originalPosition; } } using var buffer = new MemoryStream(); content.CopyTo(buffer); return new ByteArrayContent(buffer.ToArray()); } private static async Task ReadErrorAsync(HttpResponseMessage response, CancellationToken cancellationToken) { if (response.Content is null) { return string.Empty; } var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); if (string.IsNullOrWhiteSpace(text)) { return string.Empty; } var trimmed = text.Trim(); return trimmed.Length <= 512 ? trimmed : trimmed[..512]; } private static string? NormalizeHash(string? hash) => hash?.Trim().ToLowerInvariant(); private static string SanitizeFileName(string value) => string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant(); }