Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,89 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Client;
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight HTTP client for retrieving mirror index and domain artefacts.
|
||||
/// </summary>
|
||||
public sealed class MirrorManifestClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip
|
||||
};
|
||||
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly ILogger<MirrorManifestClient> _logger;
|
||||
|
||||
public MirrorManifestClient(HttpClient httpClient, ILogger<MirrorManifestClient> logger)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<MirrorIndexDocument> GetIndexAsync(string indexPath, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(indexPath))
|
||||
{
|
||||
throw new ArgumentException("Index path must be provided.", nameof(indexPath));
|
||||
}
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, indexPath);
|
||||
using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await EnsureSuccessAsync(response, indexPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var document = await JsonSerializer.DeserializeAsync<MirrorIndexDocument>(stream, JsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror index payload was empty.");
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public async Task<byte[]> DownloadAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
throw new ArgumentException("Path must be provided.", nameof(path));
|
||||
}
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, path);
|
||||
using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await EnsureSuccessAsync(response, path, cancellationToken).ConfigureAwait(false);
|
||||
return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task EnsureSuccessAsync(HttpResponseMessage response, string path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var status = (int)response.StatusCode;
|
||||
var body = string.Empty;
|
||||
|
||||
if (response.Content.Headers.ContentLength is long length && length > 0)
|
||||
{
|
||||
body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Mirror request to {Path} failed with {StatusCode}. Body: {Body}",
|
||||
path,
|
||||
status,
|
||||
string.IsNullOrEmpty(body) ? "<empty>" : body);
|
||||
|
||||
throw new HttpRequestException($"Mirror request to '{path}' failed with status {(HttpStatusCode)status} ({status}).", null, response.StatusCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal;
|
||||
|
||||
internal static class MirrorAdvisoryMapper
|
||||
{
|
||||
private const string MirrorProvenanceKind = "map";
|
||||
|
||||
private static readonly string[] TopLevelFieldMask =
|
||||
{
|
||||
ProvenanceFieldMasks.Advisory,
|
||||
ProvenanceFieldMasks.References,
|
||||
ProvenanceFieldMasks.Credits,
|
||||
ProvenanceFieldMasks.CvssMetrics,
|
||||
ProvenanceFieldMasks.Weaknesses,
|
||||
};
|
||||
|
||||
public static ImmutableArray<Advisory> Map(MirrorBundleDocument bundle)
|
||||
{
|
||||
if (bundle?.Advisories is null || bundle.Advisories.Count == 0)
|
||||
{
|
||||
return ImmutableArray<Advisory>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<Advisory>(bundle.Advisories.Count);
|
||||
var recordedAt = bundle.GeneratedAt.ToUniversalTime();
|
||||
var mirrorValue = BuildMirrorValue(bundle, recordedAt);
|
||||
var topLevelProvenance = new AdvisoryProvenance(
|
||||
StellaOpsMirrorConnector.Source,
|
||||
MirrorProvenanceKind,
|
||||
mirrorValue,
|
||||
recordedAt,
|
||||
TopLevelFieldMask);
|
||||
|
||||
foreach (var advisory in bundle.Advisories)
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = CanonicalJsonSerializer.Normalize(advisory);
|
||||
var aliases = EnsureAliasCoverage(normalized);
|
||||
var provenance = EnsureProvenance(normalized.Provenance, topLevelProvenance);
|
||||
var packages = EnsurePackageProvenance(normalized.AffectedPackages, mirrorValue, recordedAt);
|
||||
|
||||
var updated = new Advisory(
|
||||
normalized.AdvisoryKey,
|
||||
normalized.Title,
|
||||
normalized.Summary,
|
||||
normalized.Language,
|
||||
normalized.Published,
|
||||
normalized.Modified,
|
||||
normalized.Severity,
|
||||
normalized.ExploitKnown,
|
||||
aliases,
|
||||
normalized.Credits,
|
||||
normalized.References,
|
||||
packages,
|
||||
normalized.CvssMetrics,
|
||||
provenance,
|
||||
normalized.Description,
|
||||
normalized.Cwes,
|
||||
normalized.CanonicalMetricId);
|
||||
|
||||
builder.Add(updated);
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static IEnumerable<string> EnsureAliasCoverage(Advisory advisory)
|
||||
{
|
||||
var aliases = new List<string>(advisory.Aliases.Length + 1);
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var alias in advisory.Aliases)
|
||||
{
|
||||
if (seen.Add(alias))
|
||||
{
|
||||
aliases.Add(alias);
|
||||
}
|
||||
}
|
||||
|
||||
if (seen.Add(advisory.AdvisoryKey))
|
||||
{
|
||||
aliases.Add(advisory.AdvisoryKey);
|
||||
}
|
||||
|
||||
return aliases;
|
||||
}
|
||||
|
||||
private static IEnumerable<AdvisoryProvenance> EnsureProvenance(
|
||||
ImmutableArray<AdvisoryProvenance> existing,
|
||||
AdvisoryProvenance mirrorProvenance)
|
||||
{
|
||||
if (!existing.IsDefaultOrEmpty
|
||||
&& existing.Any(provenance =>
|
||||
string.Equals(provenance.Source, mirrorProvenance.Source, StringComparison.Ordinal)
|
||||
&& string.Equals(provenance.Kind, mirrorProvenance.Kind, StringComparison.Ordinal)
|
||||
&& string.Equals(provenance.Value, mirrorProvenance.Value, StringComparison.Ordinal)))
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
return existing.Add(mirrorProvenance);
|
||||
}
|
||||
|
||||
private static IEnumerable<AffectedPackage> EnsurePackageProvenance(
|
||||
ImmutableArray<AffectedPackage> packages,
|
||||
string mirrorValue,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
if (packages.IsDefaultOrEmpty || packages.Length == 0)
|
||||
{
|
||||
return packages;
|
||||
}
|
||||
|
||||
var results = new List<AffectedPackage>(packages.Length);
|
||||
|
||||
foreach (var package in packages)
|
||||
{
|
||||
var value = $"{mirrorValue};package={package.Identifier}";
|
||||
if (!package.Provenance.IsDefaultOrEmpty
|
||||
&& package.Provenance.Any(provenance =>
|
||||
string.Equals(provenance.Source, StellaOpsMirrorConnector.Source, StringComparison.Ordinal)
|
||||
&& string.Equals(provenance.Kind, MirrorProvenanceKind, StringComparison.Ordinal)
|
||||
&& string.Equals(provenance.Value, value, StringComparison.Ordinal)))
|
||||
{
|
||||
results.Add(package);
|
||||
continue;
|
||||
}
|
||||
|
||||
var masks = BuildPackageFieldMask(package);
|
||||
var packageProvenance = new AdvisoryProvenance(
|
||||
StellaOpsMirrorConnector.Source,
|
||||
MirrorProvenanceKind,
|
||||
value,
|
||||
recordedAt,
|
||||
masks);
|
||||
|
||||
var provenance = package.Provenance.Add(packageProvenance);
|
||||
var updated = new AffectedPackage(
|
||||
package.Type,
|
||||
package.Identifier,
|
||||
package.Platform,
|
||||
package.VersionRanges,
|
||||
package.Statuses,
|
||||
provenance,
|
||||
package.NormalizedVersions);
|
||||
|
||||
results.Add(updated);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static string[] BuildPackageFieldMask(AffectedPackage package)
|
||||
{
|
||||
var masks = new HashSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
ProvenanceFieldMasks.AffectedPackages,
|
||||
};
|
||||
|
||||
if (!package.VersionRanges.IsDefaultOrEmpty && package.VersionRanges.Length > 0)
|
||||
{
|
||||
masks.Add(ProvenanceFieldMasks.VersionRanges);
|
||||
}
|
||||
|
||||
if (!package.Statuses.IsDefaultOrEmpty && package.Statuses.Length > 0)
|
||||
{
|
||||
masks.Add(ProvenanceFieldMasks.PackageStatuses);
|
||||
}
|
||||
|
||||
if (!package.NormalizedVersions.IsDefaultOrEmpty && package.NormalizedVersions.Length > 0)
|
||||
{
|
||||
masks.Add(ProvenanceFieldMasks.NormalizedVersions);
|
||||
}
|
||||
|
||||
return masks.ToArray();
|
||||
}
|
||||
|
||||
private static string BuildMirrorValue(MirrorBundleDocument bundle, DateTimeOffset recordedAt)
|
||||
{
|
||||
var segments = new List<string>
|
||||
{
|
||||
$"domain={bundle.DomainId}",
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(bundle.TargetRepository))
|
||||
{
|
||||
segments.Add($"repository={bundle.TargetRepository}");
|
||||
}
|
||||
|
||||
segments.Add($"generated={recordedAt.ToString("O", CultureInfo.InvariantCulture)}");
|
||||
return string.Join(';', segments);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal;
|
||||
|
||||
public sealed record MirrorBundleDocument(
|
||||
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
|
||||
[property: JsonPropertyName("targetRepository")] string? TargetRepository,
|
||||
[property: JsonPropertyName("domainId")] string DomainId,
|
||||
[property: JsonPropertyName("displayName")] string DisplayName,
|
||||
[property: JsonPropertyName("advisoryCount")] int AdvisoryCount,
|
||||
[property: JsonPropertyName("advisories")] IReadOnlyList<Advisory> Advisories,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<MirrorSourceSummary> Sources);
|
||||
@@ -0,0 +1,38 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal;
|
||||
|
||||
public sealed record MirrorIndexDocument(
|
||||
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
|
||||
[property: JsonPropertyName("targetRepository")] string? TargetRepository,
|
||||
[property: JsonPropertyName("domains")] IReadOnlyList<MirrorIndexDomainEntry> Domains);
|
||||
|
||||
public sealed record MirrorIndexDomainEntry(
|
||||
[property: JsonPropertyName("domainId")] string DomainId,
|
||||
[property: JsonPropertyName("displayName")] string DisplayName,
|
||||
[property: JsonPropertyName("advisoryCount")] int AdvisoryCount,
|
||||
[property: JsonPropertyName("manifest")] MirrorFileDescriptor Manifest,
|
||||
[property: JsonPropertyName("bundle")] MirrorFileDescriptor Bundle,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<MirrorSourceSummary> Sources);
|
||||
|
||||
public sealed record MirrorFileDescriptor(
|
||||
[property: JsonPropertyName("path")] string Path,
|
||||
[property: JsonPropertyName("sizeBytes")] long SizeBytes,
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("signature")] MirrorSignatureDescriptor? Signature);
|
||||
|
||||
public sealed record MirrorSignatureDescriptor(
|
||||
[property: JsonPropertyName("path")] string Path,
|
||||
[property: JsonPropertyName("algorithm")] string Algorithm,
|
||||
[property: JsonPropertyName("keyId")] string KeyId,
|
||||
[property: JsonPropertyName("provider")] string Provider,
|
||||
[property: JsonPropertyName("signedAt")] DateTimeOffset SignedAt);
|
||||
|
||||
public sealed record MirrorSourceSummary(
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("firstRecordedAt")] DateTimeOffset? FirstRecordedAt,
|
||||
[property: JsonPropertyName("lastRecordedAt")] DateTimeOffset? LastRecordedAt,
|
||||
[property: JsonPropertyName("advisoryCount")] int AdvisoryCount);
|
||||
@@ -0,0 +1,125 @@
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal;
|
||||
|
||||
internal sealed record StellaOpsMirrorCursor(
|
||||
string? ExportId,
|
||||
string? BundleDigest,
|
||||
DateTimeOffset? GeneratedAt,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
string? CompletedFingerprint)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
|
||||
public static StellaOpsMirrorCursor Empty { get; } = new(
|
||||
ExportId: null,
|
||||
BundleDigest: null,
|
||||
GeneratedAt: null,
|
||||
PendingDocuments: EmptyGuids,
|
||||
PendingMappings: EmptyGuids,
|
||||
CompletedFingerprint: null);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ExportId))
|
||||
{
|
||||
document["exportId"] = ExportId;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(BundleDigest))
|
||||
{
|
||||
document["bundleDigest"] = BundleDigest;
|
||||
}
|
||||
|
||||
if (GeneratedAt.HasValue)
|
||||
{
|
||||
document["generatedAt"] = GeneratedAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(CompletedFingerprint))
|
||||
{
|
||||
document["completedFingerprint"] = CompletedFingerprint;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static StellaOpsMirrorCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var exportId = document.TryGetValue("exportId", out var exportValue) && exportValue.IsString ? exportValue.AsString : null;
|
||||
var digest = document.TryGetValue("bundleDigest", out var digestValue) && digestValue.IsString ? digestValue.AsString : null;
|
||||
DateTimeOffset? generatedAt = null;
|
||||
if (document.TryGetValue("generatedAt", out var generatedValue))
|
||||
{
|
||||
generatedAt = generatedValue.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(generatedValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(generatedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
var fingerprint = document.TryGetValue("completedFingerprint", out var fingerprintValue) && fingerprintValue.IsString
|
||||
? fingerprintValue.AsString
|
||||
: null;
|
||||
|
||||
return new StellaOpsMirrorCursor(exportId, digest, generatedAt, pendingDocuments, pendingMappings, fingerprint);
|
||||
}
|
||||
|
||||
public StellaOpsMirrorCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuids };
|
||||
|
||||
public StellaOpsMirrorCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuids };
|
||||
|
||||
public StellaOpsMirrorCursor WithBundleSnapshot(string? exportId, string? digest, DateTimeOffset generatedAt)
|
||||
=> this with
|
||||
{
|
||||
ExportId = string.IsNullOrWhiteSpace(exportId) ? ExportId : exportId,
|
||||
BundleDigest = digest,
|
||||
GeneratedAt = generatedAt,
|
||||
};
|
||||
|
||||
public StellaOpsMirrorCursor WithCompletedFingerprint(string? fingerprint)
|
||||
=> this with { CompletedFingerprint = string.IsNullOrWhiteSpace(fingerprint) ? null : fingerprint };
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror;
|
||||
|
||||
internal static class StellaOpsMirrorJobKinds
|
||||
{
|
||||
public const string Fetch = "source:stellaops-mirror:fetch";
|
||||
public const string Parse = "source:stellaops-mirror:parse";
|
||||
public const string Map = "source:stellaops-mirror:map";
|
||||
}
|
||||
|
||||
internal sealed class StellaOpsMirrorFetchJob : IJob
|
||||
{
|
||||
private readonly StellaOpsMirrorConnector _connector;
|
||||
|
||||
public StellaOpsMirrorFetchJob(StellaOpsMirrorConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class StellaOpsMirrorParseJob : IJob
|
||||
{
|
||||
private readonly StellaOpsMirrorConnector _connector;
|
||||
|
||||
public StellaOpsMirrorParseJob(StellaOpsMirrorConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class StellaOpsMirrorMapJob : IJob
|
||||
{
|
||||
private readonly StellaOpsMirrorConnector _connector;
|
||||
|
||||
public StellaOpsMirrorMapJob(StellaOpsMirrorConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.StellaOpsMirror.Tests")]
|
||||
@@ -0,0 +1,273 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.IdentityModel.Tokens;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Security;
|
||||
|
||||
/// <summary>
|
||||
/// Validates detached JWS signatures emitted by mirror bundles.
|
||||
/// </summary>
|
||||
public sealed class MirrorSignatureVerifier
|
||||
{
|
||||
private const string CachePrefix = "stellaops:mirror:public-key:";
|
||||
private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
private readonly ICryptoProviderRegistry _providerRegistry;
|
||||
private readonly ILogger<MirrorSignatureVerifier> _logger;
|
||||
private readonly IMemoryCache? _memoryCache;
|
||||
|
||||
public MirrorSignatureVerifier(
|
||||
ICryptoProviderRegistry providerRegistry,
|
||||
ILogger<MirrorSignatureVerifier> logger,
|
||||
IMemoryCache? memoryCache = null)
|
||||
{
|
||||
_providerRegistry = providerRegistry ?? throw new ArgumentNullException(nameof(providerRegistry));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_memoryCache = memoryCache;
|
||||
}
|
||||
|
||||
public Task VerifyAsync(ReadOnlyMemory<byte> payload, string signatureValue, CancellationToken cancellationToken)
|
||||
=> VerifyAsync(payload, signatureValue, expectedKeyId: null, expectedProvider: null, fallbackPublicKeyPath: null, cancellationToken);
|
||||
|
||||
public async Task VerifyAsync(
|
||||
ReadOnlyMemory<byte> payload,
|
||||
string signatureValue,
|
||||
string? expectedKeyId,
|
||||
string? expectedProvider,
|
||||
string? fallbackPublicKeyPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
throw new ArgumentException("Payload must not be empty.", nameof(payload));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(signatureValue))
|
||||
{
|
||||
throw new ArgumentException("Signature value must be provided.", nameof(signatureValue));
|
||||
}
|
||||
|
||||
if (!TryParseDetachedJws(signatureValue, out var encodedHeader, out var encodedSignature))
|
||||
{
|
||||
throw new InvalidOperationException("Detached JWS signature is malformed.");
|
||||
}
|
||||
|
||||
var headerJson = Encoding.UTF8.GetString(Base64UrlEncoder.DecodeBytes(encodedHeader));
|
||||
var header = JsonSerializer.Deserialize<MirrorSignatureHeader>(headerJson, HeaderSerializerOptions)
|
||||
?? throw new InvalidOperationException("Detached JWS header could not be parsed.");
|
||||
|
||||
if (!header.Critical.Contains("b64", StringComparer.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("Detached JWS header is missing required 'b64' critical parameter.");
|
||||
}
|
||||
|
||||
if (header.Base64Payload)
|
||||
{
|
||||
throw new InvalidOperationException("Detached JWS header sets b64=true; expected unencoded payload.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(header.KeyId))
|
||||
{
|
||||
throw new InvalidOperationException("Detached JWS header missing key identifier.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(header.Algorithm))
|
||||
{
|
||||
throw new InvalidOperationException("Detached JWS header missing algorithm identifier.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(expectedKeyId) &&
|
||||
!string.Equals(header.KeyId, expectedKeyId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Mirror bundle signature key '{header.KeyId}' did not match expected key '{expectedKeyId}'.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(expectedProvider) &&
|
||||
!string.Equals(header.Provider, expectedProvider, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Mirror bundle signature provider '{header.Provider ?? "<null>"}' did not match expected provider '{expectedProvider}'.");
|
||||
}
|
||||
|
||||
var signingInput = BuildSigningInput(encodedHeader, payload.Span);
|
||||
var signatureBytes = Base64UrlEncoder.DecodeBytes(encodedSignature);
|
||||
|
||||
var keyReference = new CryptoKeyReference(header.KeyId, header.Provider);
|
||||
CryptoSignerResolution? resolution = null;
|
||||
bool providerVerified = false;
|
||||
try
|
||||
{
|
||||
resolution = _providerRegistry.ResolveSigner(
|
||||
CryptoCapability.Verification,
|
||||
header.Algorithm,
|
||||
keyReference,
|
||||
header.Provider);
|
||||
providerVerified = await resolution.Signer.VerifyAsync(signingInput, signatureBytes, cancellationToken).ConfigureAwait(false);
|
||||
if (providerVerified)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Detached JWS verification failed for key {KeyId} via provider {Provider}.",
|
||||
header.KeyId,
|
||||
resolution.ProviderName);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Unable to resolve signer for mirror signature key {KeyId} via provider {Provider}.", header.KeyId, header.Provider ?? "<null>");
|
||||
}
|
||||
|
||||
if (providerVerified)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(fallbackPublicKeyPath) &&
|
||||
await TryVerifyWithFallbackAsync(signingInput, signatureBytes, header.Algorithm, fallbackPublicKeyPath!, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Detached JWS verification succeeded for key {KeyId} using fallback public key at {Path}.",
|
||||
header.KeyId,
|
||||
fallbackPublicKeyPath);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Detached JWS signature verification failed.");
|
||||
}
|
||||
|
||||
private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature)
|
||||
{
|
||||
var parts = value.Split("..", StringSplitOptions.None);
|
||||
if (parts.Length != 2 || string.IsNullOrEmpty(parts[0]) || string.IsNullOrEmpty(parts[1]))
|
||||
{
|
||||
encodedHeader = string.Empty;
|
||||
encodedSignature = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
encodedHeader = parts[0];
|
||||
encodedSignature = parts[1];
|
||||
return true;
|
||||
}
|
||||
|
||||
private static ReadOnlyMemory<byte> BuildSigningInput(string encodedHeader, ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var headerBytes = Encoding.ASCII.GetBytes(encodedHeader);
|
||||
var buffer = new byte[headerBytes.Length + 1 + payload.Length];
|
||||
headerBytes.CopyTo(buffer.AsSpan());
|
||||
buffer[headerBytes.Length] = (byte)'.';
|
||||
payload.CopyTo(buffer.AsSpan(headerBytes.Length + 1));
|
||||
return buffer;
|
||||
}
|
||||
|
||||
private async Task<bool> TryVerifyWithFallbackAsync(
|
||||
ReadOnlyMemory<byte> signingInput,
|
||||
ReadOnlyMemory<byte> signature,
|
||||
string algorithm,
|
||||
string fallbackPublicKeyPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var parameters = await GetFallbackPublicKeyAsync(fallbackPublicKeyPath, cancellationToken).ConfigureAwait(false);
|
||||
if (parameters is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
using var ecdsa = ECDsa.Create();
|
||||
ecdsa.ImportParameters(parameters.Value);
|
||||
var hashAlgorithm = ResolveHashAlgorithm(algorithm);
|
||||
return ecdsa.VerifyData(signingInput.Span, signature.Span, hashAlgorithm);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException or ArgumentException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to verify mirror signature using fallback public key at {Path}.", fallbackPublicKeyPath);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private Task<ECParameters?> GetFallbackPublicKeyAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (_memoryCache is null)
|
||||
{
|
||||
return Task.FromResult(LoadPublicKey(path));
|
||||
}
|
||||
|
||||
if (_memoryCache.TryGetValue<Lazy<ECParameters?>>(CachePrefix + path, out var cached))
|
||||
{
|
||||
return Task.FromResult(cached?.Value);
|
||||
}
|
||||
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
_logger.LogWarning("Mirror signature fallback public key path {Path} was not found.", path);
|
||||
return Task.FromResult<ECParameters?>(null);
|
||||
}
|
||||
|
||||
var lazy = new Lazy<ECParameters?>(
|
||||
() => LoadPublicKey(path),
|
||||
LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
var options = new MemoryCacheEntryOptions
|
||||
{
|
||||
AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(6),
|
||||
SlidingExpiration = TimeSpan.FromMinutes(30),
|
||||
};
|
||||
|
||||
_memoryCache.Set(CachePrefix + path, lazy, options);
|
||||
return Task.FromResult(lazy.Value);
|
||||
}
|
||||
|
||||
private ECParameters? LoadPublicKey(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var pem = File.ReadAllText(path);
|
||||
using var ecdsa = ECDsa.Create();
|
||||
ecdsa.ImportFromPem(pem.AsSpan());
|
||||
return ecdsa.ExportParameters(includePrivateParameters: false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or CryptographicException or ArgumentException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load mirror fallback public key from {Path}.", path);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static HashAlgorithmName ResolveHashAlgorithm(string algorithmId)
|
||||
=> algorithmId switch
|
||||
{
|
||||
{ } alg when string.Equals(alg, SignatureAlgorithms.Es256, StringComparison.OrdinalIgnoreCase) => HashAlgorithmName.SHA256,
|
||||
{ } alg when string.Equals(alg, SignatureAlgorithms.Es384, StringComparison.OrdinalIgnoreCase) => HashAlgorithmName.SHA384,
|
||||
{ } alg when string.Equals(alg, SignatureAlgorithms.Es512, StringComparison.OrdinalIgnoreCase) => HashAlgorithmName.SHA512,
|
||||
_ => throw new InvalidOperationException($"Unsupported mirror signature algorithm '{algorithmId}'."),
|
||||
};
|
||||
|
||||
private sealed record MirrorSignatureHeader(
|
||||
[property: JsonPropertyName("alg")] string Algorithm,
|
||||
[property: JsonPropertyName("kid")] string KeyId,
|
||||
[property: JsonPropertyName("provider")] string? Provider,
|
||||
[property: JsonPropertyName("typ")] string? Type,
|
||||
[property: JsonPropertyName("b64")] bool Base64Payload,
|
||||
[property: JsonPropertyName("crit")] string[] Critical);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
using System;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror.Settings;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the StellaOps mirror connector HTTP client.
|
||||
/// </summary>
|
||||
public sealed class StellaOpsMirrorConnectorOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Base address of the mirror distribution endpoint (e.g., https://mirror.stella-ops.org).
|
||||
/// </summary>
|
||||
[Required]
|
||||
public Uri BaseAddress { get; set; } = new("https://mirror.stella-ops.org", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Relative path to the mirror index document. Defaults to <c>/concelier/exports/index.json</c>.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string IndexPath { get; set; } = "/concelier/exports/index.json";
|
||||
|
||||
/// <summary>
|
||||
/// Preferred mirror domain identifier when multiple domains are published in the index.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string DomainId { get; set; } = "primary";
|
||||
|
||||
/// <summary>
|
||||
/// Maximum duration to wait on HTTP requests.
|
||||
/// </summary>
|
||||
public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Signature verification configuration for downloaded bundles.
|
||||
/// </summary>
|
||||
public SignatureOptions Signature { get; set; } = new();
|
||||
|
||||
public sealed class SignatureOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// When <c>true</c>, downloaded bundles must include a detached JWS that validates successfully.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Expected signing key identifier (kid) emitted in the detached JWS header.
|
||||
/// </summary>
|
||||
public string KeyId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Optional crypto provider hint used to resolve verification keys.
|
||||
/// </summary>
|
||||
public string? Provider { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional path to a PEM-encoded EC public key used to verify signatures when registry resolution fails.
|
||||
/// </summary>
|
||||
public string? PublicKeyPath { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,573 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Client;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Internal;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Security;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Settings;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror;
|
||||
|
||||
public sealed class StellaOpsMirrorConnector : IFeedConnector
|
||||
{
|
||||
public const string Source = "stellaops-mirror";
|
||||
private const string BundleDtoSchemaVersion = "stellaops.mirror.bundle.v1";
|
||||
|
||||
private readonly MirrorManifestClient _client;
|
||||
private readonly MirrorSignatureVerifier _signatureVerifier;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<StellaOpsMirrorConnector> _logger;
|
||||
private readonly StellaOpsMirrorConnectorOptions _options;
|
||||
|
||||
public StellaOpsMirrorConnector(
|
||||
MirrorManifestClient client,
|
||||
MirrorSignatureVerifier signatureVerifier,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<StellaOpsMirrorConnectorOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<StellaOpsMirrorConnector> logger)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
ValidateOptions(_options);
|
||||
}
|
||||
|
||||
public string SourceName => Source;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
_ = services ?? throw new ArgumentNullException(nameof(services));
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
|
||||
MirrorIndexDocument index;
|
||||
try
|
||||
{
|
||||
index = await _client.GetIndexAsync(_options.IndexPath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _stateRepository.MarkFailureAsync(Source, now, TimeSpan.FromMinutes(15), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
var domain = index.Domains.FirstOrDefault(entry =>
|
||||
string.Equals(entry.DomainId, _options.DomainId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (domain is null)
|
||||
{
|
||||
var message = $"Mirror domain '{_options.DomainId}' not present in index.";
|
||||
await _stateRepository.MarkFailureAsync(Source, now, TimeSpan.FromMinutes(30), message, cancellationToken).ConfigureAwait(false);
|
||||
throw new InvalidOperationException(message);
|
||||
}
|
||||
|
||||
var fingerprint = CreateFingerprint(index, domain);
|
||||
var isNewDigest = !string.Equals(domain.Bundle.Digest, cursor.BundleDigest, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (isNewDigest)
|
||||
{
|
||||
pendingDocuments.Clear();
|
||||
pendingMappings.Clear();
|
||||
}
|
||||
|
||||
if (string.Equals(domain.Bundle.Digest, cursor.BundleDigest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("Mirror bundle digest {Digest} unchanged; skipping fetch.", domain.Bundle.Digest);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessDomainAsync(index, domain, pendingDocuments, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _stateRepository.MarkFailureAsync(Source, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
var completedFingerprint = isNewDigest ? null : cursor.CompletedFingerprint;
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithBundleSnapshot(domain.Bundle.Path, domain.Bundle.Digest, index.GeneratedAt)
|
||||
.WithCompletedFingerprint(completedFingerprint);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
return ParseInternalAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
return MapInternalAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task ProcessDomainAsync(
|
||||
MirrorIndexDocument index,
|
||||
MirrorIndexDomainEntry domain,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var manifestBytes = await _client.DownloadAsync(domain.Manifest.Path, cancellationToken).ConfigureAwait(false);
|
||||
var bundleBytes = await _client.DownloadAsync(domain.Bundle.Path, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
VerifyDigest(domain.Manifest.Digest, manifestBytes, domain.Manifest.Path);
|
||||
VerifyDigest(domain.Bundle.Digest, bundleBytes, domain.Bundle.Path);
|
||||
|
||||
if (_options.Signature.Enabled)
|
||||
{
|
||||
if (domain.Bundle.Signature is null)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror bundle did not include a signature descriptor while verification is enabled.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_options.Signature.KeyId) &&
|
||||
!string.Equals(domain.Bundle.Signature.KeyId, _options.Signature.KeyId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Mirror bundle signature key '{domain.Bundle.Signature.KeyId}' did not match expected key '{_options.Signature.KeyId}'.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_options.Signature.Provider) &&
|
||||
!string.Equals(domain.Bundle.Signature.Provider, _options.Signature.Provider, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Mirror bundle signature provider '{domain.Bundle.Signature.Provider ?? "<null>"}' did not match expected provider '{_options.Signature.Provider}'.");
|
||||
}
|
||||
|
||||
var signatureBytes = await _client.DownloadAsync(domain.Bundle.Signature.Path, cancellationToken).ConfigureAwait(false);
|
||||
var signatureValue = Encoding.UTF8.GetString(signatureBytes).Trim();
|
||||
await _signatureVerifier.VerifyAsync(
|
||||
bundleBytes,
|
||||
signatureValue,
|
||||
expectedKeyId: _options.Signature.KeyId,
|
||||
expectedProvider: _options.Signature.Provider,
|
||||
fallbackPublicKeyPath: _options.Signature.PublicKeyPath,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else if (domain.Bundle.Signature is not null)
|
||||
{
|
||||
_logger.LogInformation("Mirror bundle provided signature descriptor but verification is disabled; skipping verification.");
|
||||
}
|
||||
|
||||
await StoreAsync(domain, index.GeneratedAt, domain.Manifest, manifestBytes, "application/json", DocumentStatuses.Mapped, addToPending: false, pendingDocuments, cancellationToken).ConfigureAwait(false);
|
||||
var bundleRecord = await StoreAsync(domain, index.GeneratedAt, domain.Bundle, bundleBytes, "application/json", DocumentStatuses.PendingParse, addToPending: true, pendingDocuments, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Stored mirror bundle {Uri} as document {DocumentId} with digest {Digest}.",
|
||||
bundleRecord.Uri,
|
||||
bundleRecord.Id,
|
||||
bundleRecord.Sha256);
|
||||
}
|
||||
|
||||
private async Task<DocumentRecord> StoreAsync(
|
||||
MirrorIndexDomainEntry domain,
|
||||
DateTimeOffset generatedAt,
|
||||
MirrorFileDescriptor descriptor,
|
||||
byte[] payload,
|
||||
string contentType,
|
||||
string status,
|
||||
bool addToPending,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var absolute = ResolveAbsolutePath(descriptor.Path);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(Source, absolute, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && string.Equals(existing.Sha256, NormalizeDigest(descriptor.Digest), StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (addToPending)
|
||||
{
|
||||
pendingDocuments.Add(existing.Id);
|
||||
}
|
||||
|
||||
return existing;
|
||||
}
|
||||
|
||||
var gridFsId = await _rawDocumentStorage.UploadAsync(Source, absolute, payload, contentType, cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var sha = ComputeSha256(payload);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["mirror.domainId"] = domain.DomainId,
|
||||
["mirror.displayName"] = domain.DisplayName,
|
||||
["mirror.path"] = descriptor.Path,
|
||||
["mirror.digest"] = NormalizeDigest(descriptor.Digest),
|
||||
["mirror.type"] = ReferenceEquals(descriptor, domain.Bundle) ? "bundle" : "manifest",
|
||||
};
|
||||
|
||||
var record = new DocumentRecord(
|
||||
existing?.Id ?? Guid.NewGuid(),
|
||||
Source,
|
||||
absolute,
|
||||
now,
|
||||
sha,
|
||||
status,
|
||||
contentType,
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: null,
|
||||
LastModified: generatedAt,
|
||||
GridFsId: gridFsId,
|
||||
ExpiresAt: null);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (addToPending)
|
||||
{
|
||||
pendingDocuments.Add(upserted.Id);
|
||||
}
|
||||
|
||||
return upserted;
|
||||
}
|
||||
|
||||
private string ResolveAbsolutePath(string path)
|
||||
{
|
||||
var uri = new Uri(_options.BaseAddress, path);
|
||||
return uri.ToString();
|
||||
}
|
||||
|
||||
private async Task<StellaOpsMirrorCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(Source, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? StellaOpsMirrorCursor.Empty : StellaOpsMirrorCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(StellaOpsMirrorCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.UpdateCursorAsync(Source, document, now, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static void VerifyDigest(string expected, ReadOnlySpan<byte> payload, string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(expected))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!expected.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Unsupported digest '{expected}' for '{path}'.");
|
||||
}
|
||||
|
||||
var actualHash = SHA256.HashData(payload);
|
||||
var actual = "sha256:" + Convert.ToHexString(actualHash).ToLowerInvariant();
|
||||
if (!string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Digest mismatch for '{path}'. Expected {expected}, computed {actual}.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var hash = SHA256.HashData(payload);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
|
||||
? digest[7..]
|
||||
: digest.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string? CreateFingerprint(MirrorIndexDocument index, MirrorIndexDomainEntry domain)
|
||||
=> CreateFingerprint(domain.Bundle.Digest, index.GeneratedAt);
|
||||
|
||||
private static string? CreateFingerprint(string? digest, DateTimeOffset? generatedAt)
|
||||
{
|
||||
var normalizedDigest = NormalizeDigest(digest ?? string.Empty);
|
||||
if (string.IsNullOrWhiteSpace(normalizedDigest) || generatedAt is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return FormattableString.Invariant($"{normalizedDigest}:{generatedAt.Value.ToUniversalTime():O}");
|
||||
}
|
||||
|
||||
private static void ValidateOptions(StellaOpsMirrorConnectorOptions options)
|
||||
{
|
||||
if (options.BaseAddress is null || !options.BaseAddress.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror connector requires an absolute baseAddress.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.DomainId))
|
||||
{
|
||||
throw new InvalidOperationException("Mirror connector requires domainId to be specified.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ParseInternalAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var parsed = 0;
|
||||
var failures = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments.ToArray())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Mirror bundle document {DocumentId} missing GridFS payload.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Mirror bundle {DocumentId} failed to download from raw storage.", documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
MirrorBundleDocument? bundle;
|
||||
string json;
|
||||
try
|
||||
{
|
||||
json = Encoding.UTF8.GetString(payload);
|
||||
bundle = CanonicalJsonSerializer.Deserialize<MirrorBundleDocument>(json);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Mirror bundle {DocumentId} failed to deserialize.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (bundle is null || bundle.Advisories is null)
|
||||
{
|
||||
_logger.LogWarning("Mirror bundle {DocumentId} produced null payload.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoBson = BsonDocument.Parse(json);
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, Source, BundleDtoSchemaVersion, dtoBson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Add(document.Id);
|
||||
parsed++;
|
||||
|
||||
_logger.LogDebug(
|
||||
"Parsed mirror bundle {DocumentId} domain={DomainId} advisories={AdvisoryCount}.",
|
||||
document.Id,
|
||||
bundle.DomainId,
|
||||
bundle.AdvisoryCount);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (parsed > 0 || failures > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Mirror parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}.",
|
||||
parsed,
|
||||
failures,
|
||||
pendingDocuments.Count,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MapInternalAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var mapped = 0;
|
||||
var failures = 0;
|
||||
var completedFingerprint = cursor.CompletedFingerprint;
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings.ToArray())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
_logger.LogWarning("Mirror document {DocumentId} missing DTO payload.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
MirrorBundleDocument? bundle;
|
||||
try
|
||||
{
|
||||
var json = dtoRecord.Payload.ToJson();
|
||||
bundle = CanonicalJsonSerializer.Deserialize<MirrorBundleDocument>(json);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Mirror DTO for document {DocumentId} failed to deserialize.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (bundle is null || bundle.Advisories is null)
|
||||
{
|
||||
_logger.LogWarning("Mirror bundle DTO {DocumentId} evaluated to null.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisories = MirrorAdvisoryMapper.Map(bundle);
|
||||
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
mapped++;
|
||||
|
||||
_logger.LogDebug(
|
||||
"Mirror map completed for document {DocumentId} domain={DomainId} advisories={AdvisoryCount}.",
|
||||
document.Id,
|
||||
bundle.DomainId,
|
||||
advisories.Length);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Mirror mapping failed for document {DocumentId}.", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingMappings.Count == 0 && failures == 0)
|
||||
{
|
||||
var fingerprint = CreateFingerprint(cursor.BundleDigest, cursor.GeneratedAt);
|
||||
if (!string.IsNullOrWhiteSpace(fingerprint))
|
||||
{
|
||||
completedFingerprint = fingerprint;
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithCompletedFingerprint(completedFingerprint);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (mapped > 0 || failures > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Mirror map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}.",
|
||||
mapped,
|
||||
failures,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
file static class UriExtensions
|
||||
{
|
||||
public static Uri Combine(this Uri baseUri, string relative)
|
||||
=> new(baseUri, relative);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror;
|
||||
|
||||
public sealed class StellaOpsMirrorConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = StellaOpsMirrorConnector.Source;
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<StellaOpsMirrorConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Client;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Security;
|
||||
using StellaOps.Concelier.Connector.StellaOpsMirror.Settings;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.DependencyInjection;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.StellaOpsMirror;
|
||||
|
||||
public sealed class StellaOpsMirrorDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:stellaopsMirror";
|
||||
private const string HttpClientName = "stellaops-mirror";
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(4);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddOptions<StellaOpsMirrorConnectorOptions>()
|
||||
.Bind(configuration.GetSection(ConfigurationSection))
|
||||
.PostConfigure(options =>
|
||||
{
|
||||
if (options.BaseAddress is null)
|
||||
{
|
||||
throw new InvalidOperationException("stellaopsMirror.baseAddress must be configured.");
|
||||
}
|
||||
})
|
||||
.ValidateOnStart();
|
||||
|
||||
services.AddSourceCommon();
|
||||
services.AddMemoryCache();
|
||||
|
||||
services.AddHttpClient(HttpClientName, (sp, client) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<StellaOpsMirrorConnectorOptions>>().Value;
|
||||
client.BaseAddress = options.BaseAddress;
|
||||
client.Timeout = options.HttpTimeout;
|
||||
client.DefaultRequestHeaders.Accept.Clear();
|
||||
client.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
|
||||
});
|
||||
|
||||
services.AddTransient<MirrorManifestClient>(sp =>
|
||||
{
|
||||
var factory = sp.GetRequiredService<IHttpClientFactory>();
|
||||
var httpClient = factory.CreateClient(HttpClientName);
|
||||
return ActivatorUtilities.CreateInstance<MirrorManifestClient>(sp, httpClient);
|
||||
});
|
||||
|
||||
services.TryAddSingleton<MirrorSignatureVerifier>();
|
||||
services.AddTransient<StellaOpsMirrorConnector>();
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler.AddJob<StellaOpsMirrorFetchJob>(
|
||||
StellaOpsMirrorJobKinds.Fetch,
|
||||
cronExpression: "*/15 * * * *",
|
||||
timeout: FetchTimeout,
|
||||
leaseDuration: LeaseDuration);
|
||||
scheduler.AddJob<StellaOpsMirrorParseJob>(
|
||||
StellaOpsMirrorJobKinds.Parse,
|
||||
cronExpression: null,
|
||||
timeout: TimeSpan.FromMinutes(5),
|
||||
leaseDuration: LeaseDuration,
|
||||
enabled: false);
|
||||
scheduler.AddJob<StellaOpsMirrorMapJob>(
|
||||
StellaOpsMirrorJobKinds.Map,
|
||||
cronExpression: null,
|
||||
timeout: TimeSpan.FromMinutes(5),
|
||||
leaseDuration: LeaseDuration,
|
||||
enabled: false);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# StellaOps Mirror Connector Task Board (Sprint 8)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| FEEDCONN-STELLA-08-001 | DONE (2025-10-20) | BE-Conn-Stella | CONCELIER-EXPORT-08-201 | Implement Concelier mirror fetcher hitting `https://<domain>.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. | Fetch job downloads mirror manifest, verifies digest/signature, stores raw docs with tests covering happy-path + tampered manifest. *(Completed 2025-10-20: detached JWS + digest enforcement, metadata persisted, and regression coverage via `dotnet test src/Concelier/__Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj`.)* |
|
||||
| FEEDCONN-STELLA-08-002 | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | Mapper produces advisories/aliases/affected with mirror provenance; fixtures assert canonical parity with upstream JSON exporters. |
|
||||
| FEEDCONN-STELLA-08-003 | DONE (2025-10-20) | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | Connector resumes from last export, handles deletion/delta cases, docs updated with config sample; integration test covers resume + new export scenario. |
|
||||
Reference in New Issue
Block a user