Restructure solution layout by module

This commit is contained in:
master
2025-10-28 15:10:40 +02:00
parent 95daa159c4
commit d870da18ce
4103 changed files with 192899 additions and 187024 deletions

View File

@@ -0,0 +1,38 @@
using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Export;
namespace StellaOps.Excititor.ArtifactStores.S3.Extensions;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddVexS3ArtifactClient(this IServiceCollection services, Action<S3ArtifactClientOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddSingleton(CreateS3Client);
services.AddSingleton<IS3ArtifactClient, S3ArtifactClient>();
return services;
}
private static IAmazonS3 CreateS3Client(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<S3ArtifactClientOptions>>().Value;
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
ForcePathStyle = options.ForcePathStyle,
};
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
{
config.ServiceURL = options.ServiceUrl;
}
return new AmazonS3Client(config);
}
}

View File

@@ -0,0 +1,85 @@
using Amazon.S3;
using Amazon.S3.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Export;
namespace StellaOps.Excititor.ArtifactStores.S3;
public sealed class S3ArtifactClientOptions
{
public string Region { get; set; } = "us-east-1";
public string? ServiceUrl { get; set; }
= null;
public bool ForcePathStyle { get; set; }
= true;
}
public sealed class S3ArtifactClient : IS3ArtifactClient
{
private readonly IAmazonS3 _s3;
private readonly ILogger<S3ArtifactClient> _logger;
public S3ArtifactClient(IAmazonS3 s3, ILogger<S3ArtifactClient> logger)
{
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<bool> ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken)
{
try
{
var metadata = await _s3.GetObjectMetadataAsync(bucketName, key, cancellationToken).ConfigureAwait(false);
return metadata.HttpStatusCode == System.Net.HttpStatusCode.OK;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return false;
}
}
public async Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary<string, string> metadata, CancellationToken cancellationToken)
{
var request = new PutObjectRequest
{
BucketName = bucketName,
Key = key,
InputStream = content,
AutoCloseStream = false,
};
foreach (var kvp in metadata)
{
request.Metadata[kvp.Key] = kvp.Value;
}
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Uploaded object {Bucket}/{Key}", bucketName, key);
}
public async Task<Stream?> GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken)
{
try
{
var response = await _s3.GetObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false);
var buffer = new MemoryStream();
await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
return buffer;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Object {Bucket}/{Key} not found", bucketName, key);
return null;
}
}
public async Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken)
{
await _s3.DeleteObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Deleted object {Bucket}/{Key}", bucketName, key);
}
}

View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.305.6" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Export\StellaOps.Excititor.Export.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Builds and verifies in-toto/DSSE attestations for Excititor exports and integrates with Rekor v2 transparency logs.
## Scope
- Attestation envelope builders, signing workflows (keyless/keyed), and predicate model definitions.
- Rekor v2 client implementation (submit, verify, poll inclusion) with retry/backoff policies.
- Verification utilities reused by Worker for periodic revalidation.
- Configuration bindings for signer identity, Rekor endpoints, and offline bundle operation.
## Participants
- Export module calls into this layer to generate attestations after export artifacts are produced.
- WebService and Worker consume verification helpers to ensure stored envelopes remain valid.
- CLI `excititor verify` leverages verification services through WebService endpoints.
## Interfaces & contracts
- `IExportAttestor`, `ITransparencyLogClient`, predicate DTOs, and verification result records.
- Extension methods to register attestation services in DI across WebService/Worker.
## In/Out of scope
In: attestation creation, verification, Rekor integration, signer configuration.
Out: export artifact generation, storage persistence, CLI interaction layers.
## Observability & security expectations
- Structured logs for signing/verification with envelope digest, Rekor URI, and latency; never log private keys.
- Metrics for attestation successes/failures and Rekor submission durations.
## Tests
- Unit tests and integration stubs (with fake Rekor) will live in `../StellaOps.Excititor.Attestation.Tests`.

View File

@@ -0,0 +1,13 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.Attestation.Dsse;
public sealed record DsseEnvelope(
[property: JsonPropertyName("payload")] string Payload,
[property: JsonPropertyName("payloadType")] string PayloadType,
[property: JsonPropertyName("signatures")] IReadOnlyList<DsseSignature> Signatures);
public sealed record DsseSignature(
[property: JsonPropertyName("sig")] string Signature,
[property: JsonPropertyName("keyid")] string? KeyId);

View File

@@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Attestation.Dsse;
public sealed class VexDsseBuilder
{
internal const string PayloadType = "application/vnd.in-toto+json";
private readonly IVexSigner _signer;
private readonly ILogger<VexDsseBuilder> _logger;
private readonly JsonSerializerOptions _serializerOptions;
public VexDsseBuilder(IVexSigner signer, ILogger<VexDsseBuilder> logger)
{
_signer = signer ?? throw new ArgumentNullException(nameof(signer));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_serializerOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
WriteIndented = false,
};
_serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase));
}
public async ValueTask<DsseEnvelope> CreateEnvelopeAsync(
VexAttestationRequest request,
IReadOnlyDictionary<string, string>? metadata,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var predicate = VexAttestationPredicate.FromRequest(request, metadata);
var subject = new VexInTotoSubject(
request.ExportId,
new Dictionary<string, string>(StringComparer.Ordinal)
{
{ request.Artifact.Algorithm.ToLowerInvariant(), request.Artifact.Digest }
});
var statement = new VexInTotoStatement(
VexInTotoStatement.InTotoType,
"https://stella-ops.org/attestations/vex-export",
new[] { subject },
predicate);
var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(statement, _serializerOptions);
var signatureResult = await _signer.SignAsync(payloadBytes, cancellationToken).ConfigureAwait(false);
var envelope = new DsseEnvelope(
Convert.ToBase64String(payloadBytes),
PayloadType,
new[] { new DsseSignature(signatureResult.Signature, signatureResult.KeyId) });
_logger.LogDebug("DSSE envelope created for export {ExportId}", request.ExportId);
return envelope;
}
public static string ComputeEnvelopeDigest(DsseEnvelope envelope)
{
ArgumentNullException.ThrowIfNull(envelope);
var envelopeJson = JsonSerializer.Serialize(envelope, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
});
var bytes = Encoding.UTF8.GetBytes(envelopeJson);
var hash = SHA256.HashData(bytes);
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
}

View File

@@ -0,0 +1,157 @@
# EXCITITOR-ATTEST-01-003 - Verification & Observability Plan
- **Date:** 2025-10-19
- **Status:** In progress (2025-10-22)
- **Owner:** Team Excititor Attestation
- **Related tasks:** EXCITITOR-ATTEST-01-003 (Wave 0), EXCITITOR-WEB-01-003/004, EXCITITOR-WORKER-01-003
- **Prerequisites satisfied:** EXCITITOR-ATTEST-01-002 (Rekor v2 client integration)
## 1. Objectives
1. Provide deterministic attestation verification helpers consumable by Excititor WebService (`/excititor/verify`, `/excititor/export*`) and Worker re-verification loops.
2. Surface structured diagnostics for success, soft failures, and hard failures (signature mismatch, Rekor gaps, artifact digest drift).
3. Emit observability signals (logs, metrics, optional tracing) that can run offline and degrade gracefully when transparency services are unreachable.
4. Add regression tests (unit + integration) covering positive path, negative path, and offline fallback scenarios.
## 2. Deliverables
- `IVexAttestationVerifier` abstraction + `VexAttestationVerifier` implementation inside `StellaOps.Excititor.Attestation`, encapsulating DSSE validation, predicate checks, artifact digest confirmation, Rekor inclusion verification, and deterministic diagnostics.
- DI wiring (extension method) for registering verifier + instrumentation dependencies alongside the existing signer/rekor client.
- Shared `VexAttestationDiagnostics` record describing normalized diagnostic keys consumed by Worker/WebService logging.
- Metrics utility (`AttestationMetrics`) exposing counters/histograms via `System.Diagnostics.Metrics`, exported under `StellaOps.Excititor.Attestation` meter.
- Activity source (`AttestationActivitySource`) for optional tracing spans around sign/verify operations.
- Documentation updates (`EXCITITOR-ATTEST-01-003-plan.md`, `TASKS.md` notes) describing instrumentation + test expectations.
- Test coverage in `StellaOps.Excititor.Attestation.Tests` (unit) and scaffolding notes for WebService/Worker integration tests.
## 3. Verification Flow
### 3.1 Inputs
- `VexAttestationRequest` from Core (contains export identifiers, artifact digest, metadata, source providers).
- Optional Rekor reference from previous signing (`VexAttestationMetadata.Rekor`).
- Configured policies (tolerated clock skew, Rekor verification toggle, offline mode flag, maximum metadata drift).
### 3.2 Steps
1. **Envelope decode** - retrieve DSSE envelope + predicate from storage (Worker) or request payload (WebService), canonicalize JSON, compute digest, compare with metadata `envelopeDigest`.
2. **Subject validation** - ensure subject digest matches exported artifact digest (algorithm & value) and export identifier matches `request.ExportId`.
3. **Signature verification** - delegate to signer/verifier abstraction (cosign/x509) using configured trust anchors; record `signature_state` diagnostic (verified, skipped_offline, failed).
4. **Provenance checks** - confirm predicate type (`https://stella-ops.org/attestations/vex-export`) and metadata shape; enforce deterministic timestamp tolerance.
5. **Transparency log** - if Rekor reference present and verification enabled, call `ITransparencyLogClient.VerifyAsync` with retry/backoff budget; support offline bypass with diagnostic `rekor_state=unreachable`.
6. **Result aggregation** - produce `VexAttestationVerification` containing `IsValid` flag and diagnostics map (includes `failure_reason` when invalid).
### 3.3 Failure Categories & Handling
| Category | Detection | Handling |
|---|---|---|
| Signature mismatch | Signer verification failure or subject digest mismatch | Mark invalid, emit warning log, increment `verify.failed` counter with `reason=signature_mismatch`. |
| Rekor absence/stale | Rekor verify returns false | Mark invalid unless offline mode configured; log with correlation ID; `reason=rekor_missing`. |
| Predicate schema drift | Predicate type or required fields missing | Mark invalid, include `reason=predicate_invalid`. |
| Time skew | `signedAt` older than policy threshold | Mark invalid (hard) or warn (soft) per options; include `reason=stale_attestation`. |
| Unexpected metadata | Unknown export format, provider mismatch | Mark invalid; `reason=metadata_mismatch`. |
| Offline Rekor | HTTP client throws | Mark soft failure if `AllowOfflineTransparency` true; degrade metrics with `rekor_state=offline`. |
## 4. Observability
### 4.1 Metrics (Meter name: `StellaOps.Excititor.Attestation`)
| Metric | Type | Dimensions | Description |
|---|---|---|---|
| `stellaops.excititor.attestation.verify.total` | Counter<long> | `result` (`success`/`failure`/`soft_failure`), `component` (`webservice`/`worker`), `reverify` (`true`/`false`) | Counts verification attempts. |
| `stellaops.excititor.attestation.verify.duration.ms` | Histogram<double> | `component`, `result` | Measures end-to-end verification latency. |
| `stellaops.excititor.attestation.verify.rekor.calls` | Counter<long> | `result` (`verified`/`unreachable`/`skipped`) | Rekor verification outcomes. |
| `stellaops.excititor.attestation.verify.cache.hit` | Counter<long> | `hit` (`true`/`false`) | Tracks reuse of cached verification results (Worker loop). |
Metrics must register via static helper using `Meter` and support offline operation (no exporter dependency). Histogram records double milliseconds; use `Stopwatch.GetElapsedTime` for monotonic timing.
### 4.2 Logging
- Use structured logs (`ILogger<VexAttestationVerifier>`) with event IDs: `AttestationVerified` (Information), `AttestationVerificationFailed` (Warning), `AttestationVerificationError` (Error).
- Include correlation ID (`request.QuerySignature.Value`), `exportId`, `envelopeDigest`, `rekorLocation`, `reason`, and `durationMs`.
- Avoid logging private keys or full envelope; log envelope digest only. For debug builds, gate optional envelope JSON behind `LogLevel.Trace` and configuration flag.
### 4.3 Tracing
- Activity source name `StellaOps.Excititor.Attestation` with spans `attestation.verify` (parent from WebService request or Worker job) including tags: `stellaops.export_id`, `stellaops.result`, `stellaops.rekor.state`.
- Propagate Activity through Rekor client via `HttpClient` instrumentation (auto instrumentation available).
## 5. Integration Points
### 5.1 WebService
- Inject `IVexAttestationVerifier` into export endpoints and `/excititor/verify` handler.
- Persist verification result diagnostics alongside response payload for deterministic clients.
- Return HTTP 200 with `{ valid: true }` when verified; 409 for invalid attestation with diagnostics JSON; 503 when Rekor unreachable and offline override disabled.
- Add caching for idempotent verification (e.g., by envelope digest) to reduce Rekor calls and surface via metrics.
### 5.2 Worker
- Schedule background job (`EXCITITOR-WORKER-01-003`) to re-verify stored attestations on TTL (default 12h) using new verifier; on failure, flag export for re-sign and notify via event bus (future task).
- Emit logs/metrics with `component=worker`; include job IDs and next scheduled run.
- Provide cancellation-aware loops (respect `CancellationToken`) and deterministic order (sorted by export id).
### 5.3 Storage / Cache Hooks
- Store latest verification status and diagnostics in attestation metadata collection (Mongo) keyed by `envelopeDigest` + `artifactDigest` to avoid duplicate work.
- Expose read API (via WebService) for clients to fetch last verification timestamp + result.
## 6. Test Strategy
### 6.1 Unit Tests (`StellaOps.Excititor.Attestation.Tests`)
- `VexAttestationVerifierTests.VerifyAsync_Succeeds_WhenSignatureAndRekorValid` - uses fake signer/verifier + in-memory Rekor client returning success.
- `...ReturnsSoftFailure_WhenRekorOfflineAndAllowed` - ensure `IsValid=true`, diagnostic `rekor_state=offline`, metric increments `result=soft_failure`.
- `...Fails_WhenDigestMismatch` - ensures invalid result, log entry recorded, metrics increment `result=failure` with `reason=signature_mismatch`.
- `...Fails_WhenPredicateTypeUnexpected` - invalid with `reason=predicate_invalid`.
- `...RespectsCancellation` - cancellation token triggered before Rekor call results in `OperationCanceledException` and no metrics increments beyond started attempt.
### 6.2 WebService Integration Tests (`StellaOps.Excititor.WebService.Tests`)
- `VerifyEndpoint_Returns200_OnValidAttestation` - mocks verifier to return success, asserts response payload, metrics stub invoked.
- `VerifyEndpoint_Returns409_OnInvalid` - invalid diag forwarded, ensures logging occurs.
- `ExportEndpoint_IncludesVerificationDiagnostics` - ensures signed export responses include last verification metadata.
### 6.3 Worker Tests (`StellaOps.Excititor.Worker.Tests`)
- `ReverificationJob_RequeuesOnFailure` - invalid result triggers requeue/backoff.
- `ReverificationJob_PersistsStatusAndMetrics` - success path updates repository & metrics.
### 6.4 Determinism/Regression
- Golden test verifying that identical inputs produce identical diagnostics dictionaries (sorted keys).
- Ensure metrics dimensions remain stable via snapshot test (e.g., capturing tags in fake meter listener).
## 7. Implementation Sequencing
1. Introduce verifier abstraction + implementation with basic tests (signature + Rekor success/failure).
2. Add observability helpers (metrics, activity, logging) and wire into verifier; extend tests to assert instrumentation (using in-memory listener/log sink).
3. Update WebService DI/service layer to use verifier; craft endpoint integration tests.
4. Update Worker scheduling code to call verifier & emit metrics.
5. Wire persistence/caching and document configuration knobs (retry, offline, TTL).
6. Finalize documentation (architecture updates, runbook entries) before closing task.
## 8. Configuration Defaults
- `AttestationVerificationOptions` (new): `RequireRekor=true`, `AllowOfflineTransparency=false`, `MaxClockSkew=PT5M`, `ReverifyInterval=PT12H`, `CacheWindow=PT1H`.
- Options bind from configuration section `Excititor:Attestation` across WebService/Worker; offline kit ships defaults.
## 9. Open Questions
- Should verification gracefully accept legacy predicate types (pre-1.0) or hard fail? (Proposed: allow via allowlist with warning diagnostics.)
- Do we need cross-module eventing when verification fails (e.g., notify Export module) or is logging sufficient in Wave 0? (Proposed: log + metrics now, escalate in later wave.)
- Confirm whether Worker re-verification writes to Mongo or triggers Export module to re-sign artifacts automatically; placeholder: record status + timestamp only.
## 10. Acceptance Criteria
- Plan approved by Attestation + WebService + Worker leads.
- Metrics/logging names peer-reviewed to avoid collisions.
- Test backlog items entered into respective `TASKS.md` once implementation starts.
- Documentation (this plan) linked from `TASKS.md` notes for discoverability.
## 11. 2025-10-22 Progress Notes
- Implemented `IVexAttestationVerifier`/`VexAttestationVerifier` with structural validation (subject/predicate checks, digest comparison, Rekor probes) and diagnostics map.
- Added `VexAttestationVerificationOptions` (RequireTransparencyLog, AllowOfflineTransparency, MaxClockSkew) and wired configuration through WebService DI.
- Created `VexAttestationMetrics` (`excititor.attestation.verify_total`, `excititor.attestation.verify_duration_seconds`) and hooked into verification flow with component/rekor tags.
- `VexAttestationClient.VerifyAsync` now delegates to the verifier; DI registers metrics + verifier via `AddVexAttestation`.
- Added unit coverage in `VexAttestationVerifierTests` (happy path, digest mismatch, offline Rekor) and updated client/export/webservice stubs to new verification signature.

View File

@@ -0,0 +1,27 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Attestation.Extensions;
public static class VexAttestationServiceCollectionExtensions
{
public static IServiceCollection AddVexAttestation(this IServiceCollection services)
{
services.AddSingleton<VexDsseBuilder>();
services.AddSingleton<VexAttestationMetrics>();
services.AddSingleton<IVexAttestationVerifier, VexAttestationVerifier>();
services.AddSingleton<IVexAttestationClient, VexAttestationClient>();
return services;
}
public static IServiceCollection AddVexRekorClient(this IServiceCollection services, Action<RekorHttpClientOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddHttpClient<ITransparencyLogClient, RekorHttpClient>();
return services;
}
}

View File

@@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json.Serialization;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Attestation.Models;
public sealed record VexAttestationPredicate(
string ExportId,
string QuerySignature,
string ArtifactAlgorithm,
string ArtifactDigest,
VexExportFormat Format,
DateTimeOffset CreatedAt,
IReadOnlyList<string> SourceProviders,
IReadOnlyDictionary<string, string> Metadata)
{
public static VexAttestationPredicate FromRequest(
VexAttestationRequest request,
IReadOnlyDictionary<string, string>? metadata = null)
=> new(
request.ExportId,
request.QuerySignature.Value,
request.Artifact.Algorithm,
request.Artifact.Digest,
request.Format,
request.CreatedAt,
request.SourceProviders,
metadata is null ? ImmutableDictionary<string, string>.Empty : metadata.ToImmutableDictionary(StringComparer.Ordinal));
}
public sealed record VexInTotoSubject(
string Name,
IReadOnlyDictionary<string, string> Digest);
public sealed record VexInTotoStatement(
[property: JsonPropertyName("_type")] string Type,
string PredicateType,
IReadOnlyList<VexInTotoSubject> Subject,
VexAttestationPredicate Predicate)
{
public static readonly string InTotoType = "https://in-toto.io/Statement/v0.1";
}

View File

@@ -0,0 +1,12 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Signing;
public sealed record VexSignedPayload(string Signature, string? KeyId);
public interface IVexSigner
{
ValueTask<VexSignedPayload> SignAsync(ReadOnlyMemory<byte> payload, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,9 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-ATTEST-01-001 In-toto predicate & DSSE builder|Team Excititor Attestation|EXCITITOR-CORE-01-001|**DONE (2025-10-16)** Added deterministic in-toto predicate/statement models, DSSE envelope builder wired to signer abstraction, and attestation client producing metadata + diagnostics.|
|EXCITITOR-ATTEST-01-002 Rekor v2 client integration|Team Excititor Attestation|EXCITITOR-ATTEST-01-001|**DONE (2025-10-16)** Implemented Rekor HTTP client with retry/backoff, transparency log abstraction, DI helpers, and attestation client integration capturing Rekor metadata + diagnostics.|
|EXCITITOR-ATTEST-01-003 Verification suite & observability|Team Excititor Attestation|EXCITITOR-ATTEST-01-002|DOING (2025-10-22) Continuing implementation: build `IVexAttestationVerifier`, wire metrics/logging, and add regression tests. Draft plan in `EXCITITOR-ATTEST-01-003-plan.md` (2025-10-19) guides scope; updating with worknotes as progress lands.|
> Remark (2025-10-22): Added verifier implementation + metrics/tests; next steps include wiring into WebService/Worker flows and expanding negative-path coverage.

View File

@@ -0,0 +1,14 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Excititor.Attestation.Dsse;
namespace StellaOps.Excititor.Attestation.Transparency;
public sealed record TransparencyLogEntry(string Id, string Location, string? LogIndex, string? InclusionProofUrl);
public interface ITransparencyLogClient
{
ValueTask<TransparencyLogEntry> SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken);
ValueTask<bool> VerifyAsync(string entryLocation, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,91 @@
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Attestation.Dsse;
namespace StellaOps.Excititor.Attestation.Transparency;
internal sealed class RekorHttpClient : ITransparencyLogClient
{
private readonly HttpClient _httpClient;
private readonly RekorHttpClientOptions _options;
private readonly ILogger<RekorHttpClient> _logger;
public RekorHttpClient(HttpClient httpClient, IOptions<RekorHttpClientOptions> options, ILogger<RekorHttpClient> logger)
{
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (!string.IsNullOrWhiteSpace(_options.BaseAddress))
{
_httpClient.BaseAddress = new Uri(_options.BaseAddress, UriKind.Absolute);
}
if (!string.IsNullOrWhiteSpace(_options.ApiKey))
{
_httpClient.DefaultRequestHeaders.Add("Authorization", _options.ApiKey);
}
}
public async ValueTask<TransparencyLogEntry> SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(envelope);
var payload = JsonSerializer.Serialize(envelope);
using var content = new StringContent(payload);
content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
HttpResponseMessage? response = null;
for (var attempt = 0; attempt < _options.RetryCount; attempt++)
{
response = await _httpClient.PostAsync("/api/v2/log/entries", content, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
break;
}
_logger.LogWarning("Rekor submission failed with status {Status}; attempt {Attempt}", response.StatusCode, attempt + 1);
if (attempt + 1 < _options.RetryCount)
{
await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false);
}
}
if (response is null || !response.IsSuccessStatusCode)
{
throw new HttpRequestException($"Failed to submit attestation to Rekor ({response?.StatusCode}).");
}
var entryLocation = response.Headers.Location?.ToString() ?? string.Empty;
var body = await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken: cancellationToken).ConfigureAwait(false);
var entry = ParseEntryLocation(entryLocation, body);
_logger.LogInformation("Rekor entry recorded at {Location}", entry.Location);
return entry;
}
public async ValueTask<bool> VerifyAsync(string entryLocation, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(entryLocation))
{
return false;
}
var response = await _httpClient.GetAsync(entryLocation, cancellationToken).ConfigureAwait(false);
return response.IsSuccessStatusCode;
}
private static TransparencyLogEntry ParseEntryLocation(string location, JsonElement body)
{
var id = body.TryGetProperty("uuid", out var uuid) ? uuid.GetString() ?? string.Empty : Guid.NewGuid().ToString();
var logIndex = body.TryGetProperty("logIndex", out var logIndexElement) ? logIndexElement.GetString() : null;
string? inclusionProof = null;
if (body.TryGetProperty("verification", out var verification) && verification.TryGetProperty("inclusionProof", out var inclusion))
{
inclusionProof = inclusion.GetProperty("logIndex").GetRawText();
}
return new TransparencyLogEntry(id, location, logIndex, inclusionProof);
}
}

View File

@@ -0,0 +1,13 @@
namespace StellaOps.Excititor.Attestation.Transparency;
public sealed class RekorHttpClientOptions
{
public string BaseAddress { get; set; } = "https://rekor.sigstore.dev";
public string? ApiKey { get; set; }
= null;
public int RetryCount { get; set; } = 3;
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(2);
}

View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Attestation.Verification;
public interface IVexAttestationVerifier
{
ValueTask<VexAttestationVerification> VerifyAsync(VexAttestationVerificationRequest request, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,35 @@
using System;
using System.Diagnostics.Metrics;
namespace StellaOps.Excititor.Attestation.Verification;
public sealed class VexAttestationMetrics : IDisposable
{
public const string MeterName = "StellaOps.Excititor.Attestation";
public const string MeterVersion = "1.0";
private readonly Meter _meter;
private bool _disposed;
public VexAttestationMetrics()
{
_meter = new Meter(MeterName, MeterVersion);
VerifyTotal = _meter.CreateCounter<long>("excititor.attestation.verify_total", description: "Attestation verification attempts grouped by result/component/rekor.");
VerifyDuration = _meter.CreateHistogram<double>("excititor.attestation.verify_duration_seconds", unit: "s", description: "Attestation verification latency in seconds.");
}
public Counter<long> VerifyTotal { get; }
public Histogram<double> VerifyDuration { get; }
public void Dispose()
{
if (_disposed)
{
return;
}
_meter.Dispose();
_disposed = true;
}
}

View File

@@ -0,0 +1,28 @@
using System;
namespace StellaOps.Excititor.Attestation.Verification;
public sealed class VexAttestationVerificationOptions
{
private TimeSpan _maxClockSkew = TimeSpan.FromMinutes(5);
/// <summary>
/// When true, verification fails if no transparency record is present.
/// </summary>
public bool RequireTransparencyLog { get; set; } = true;
/// <summary>
/// Allows verification to succeed when the transparency log cannot be reached.
/// A diagnostic entry is still emitted to signal the degraded state.
/// </summary>
public bool AllowOfflineTransparency { get; set; }
/// <summary>
/// Maximum tolerated clock skew between the attestation creation time and the verification context timestamp.
/// </summary>
public TimeSpan MaxClockSkew
{
get => _maxClockSkew;
set => _maxClockSkew = value < TimeSpan.Zero ? TimeSpan.Zero : value;
}
}

View File

@@ -0,0 +1,471 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Attestation.Verification;
internal sealed class VexAttestationVerifier : IVexAttestationVerifier
{
private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions StatementSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) },
};
private readonly ILogger<VexAttestationVerifier> _logger;
private readonly ITransparencyLogClient? _transparencyLogClient;
private readonly VexAttestationVerificationOptions _options;
private readonly VexAttestationMetrics _metrics;
public VexAttestationVerifier(
ILogger<VexAttestationVerifier> logger,
ITransparencyLogClient? transparencyLogClient,
IOptions<VexAttestationVerificationOptions> options,
VexAttestationMetrics metrics)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
ArgumentNullException.ThrowIfNull(options);
_transparencyLogClient = transparencyLogClient;
_options = options.Value;
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
}
public async ValueTask<VexAttestationVerification> VerifyAsync(
VexAttestationVerificationRequest request,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var stopwatch = Stopwatch.StartNew();
var diagnostics = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
var resultLabel = "valid";
var rekorState = "skipped";
var component = request.IsReverify ? "worker" : "webservice";
try
{
if (string.IsNullOrWhiteSpace(request.Envelope))
{
diagnostics["envelope.state"] = "missing";
_logger.LogWarning("Attestation envelope is missing for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!TryDeserializeEnvelope(request.Envelope, out var envelope, diagnostics))
{
_logger.LogWarning("Failed to deserialize attestation envelope for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!string.Equals(envelope.PayloadType, VexDsseBuilder.PayloadType, StringComparison.OrdinalIgnoreCase))
{
diagnostics["payload.type"] = envelope.PayloadType ?? string.Empty;
_logger.LogWarning(
"Unexpected DSSE payload type {PayloadType} for export {ExportId}",
envelope.PayloadType,
request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (envelope.Signatures is null || envelope.Signatures.Count == 0)
{
diagnostics["signature.state"] = "missing";
_logger.LogWarning("Attestation envelope for export {ExportId} does not contain signatures.", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
var payloadBase64 = envelope.Payload ?? string.Empty;
if (!TryDecodePayload(payloadBase64, out var payloadBytes, diagnostics))
{
_logger.LogWarning("Failed to decode attestation payload for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!TryDeserializeStatement(payloadBytes, out var statement, diagnostics))
{
_logger.LogWarning("Failed to deserialize DSSE statement for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidatePredicateType(statement, request, diagnostics))
{
_logger.LogWarning("Predicate type mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidateSubject(statement, request, diagnostics))
{
_logger.LogWarning("Subject mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidatePredicate(statement, request, diagnostics))
{
_logger.LogWarning("Predicate payload mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidateMetadataDigest(envelope, request.Metadata, diagnostics))
{
_logger.LogWarning("Attestation digest mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidateSignedAt(request.Metadata, request.Attestation.CreatedAt, diagnostics))
{
_logger.LogWarning("SignedAt validation failed for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
rekorState = await VerifyTransparencyAsync(request.Metadata, diagnostics, cancellationToken).ConfigureAwait(false);
if (rekorState is "missing" or "unverified" or "client_unavailable")
{
resultLabel = "invalid";
return BuildResult(false);
}
diagnostics["signature.state"] = "present";
return BuildResult(true);
}
catch (Exception ex)
{
diagnostics["error"] = ex.GetType().Name;
diagnostics["error.message"] = ex.Message;
resultLabel = "error";
_logger.LogError(ex, "Unexpected exception verifying attestation for export {ExportId}", request.Attestation.ExportId);
return BuildResult(false);
}
finally
{
stopwatch.Stop();
var tags = new KeyValuePair<string, object?>[]
{
new("result", resultLabel),
new("component", component),
new("rekor", rekorState),
};
_metrics.VerifyTotal.Add(1, tags);
_metrics.VerifyDuration.Record(stopwatch.Elapsed.TotalSeconds, tags);
}
VexAttestationVerification BuildResult(bool isValid)
{
diagnostics["result"] = resultLabel;
diagnostics["component"] = component;
diagnostics["rekor.state"] = rekorState;
return new VexAttestationVerification(isValid, diagnostics.ToImmutable());
}
}
private static bool TryDeserializeEnvelope(
string envelopeJson,
out DsseEnvelope envelope,
ImmutableDictionary<string, string>.Builder diagnostics)
{
try
{
envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson, EnvelopeSerializerOptions)
?? throw new InvalidOperationException("Envelope deserialized to null.");
return true;
}
catch (Exception ex)
{
diagnostics["envelope.error"] = ex.GetType().Name;
envelope = default!;
return false;
}
}
private static bool TryDecodePayload(
string payloadBase64,
out byte[] payloadBytes,
ImmutableDictionary<string, string>.Builder diagnostics)
{
try
{
payloadBytes = Convert.FromBase64String(payloadBase64);
return true;
}
catch (FormatException)
{
diagnostics["payload.base64"] = "invalid";
payloadBytes = Array.Empty<byte>();
return false;
}
}
private static bool TryDeserializeStatement(
byte[] payload,
out VexInTotoStatement statement,
ImmutableDictionary<string, string>.Builder diagnostics)
{
try
{
statement = JsonSerializer.Deserialize<VexInTotoStatement>(payload, StatementSerializerOptions)
?? throw new InvalidOperationException("Statement deserialized to null.");
return true;
}
catch (Exception ex)
{
diagnostics["payload.error"] = ex.GetType().Name;
statement = default!;
return false;
}
}
private static bool ValidatePredicateType(
VexInTotoStatement statement,
VexAttestationVerificationRequest request,
ImmutableDictionary<string, string>.Builder diagnostics)
{
var predicateType = statement.PredicateType ?? string.Empty;
if (!string.Equals(predicateType, request.Metadata.PredicateType, StringComparison.Ordinal))
{
diagnostics["predicate.type"] = predicateType;
return false;
}
return true;
}
private static bool ValidateSubject(
VexInTotoStatement statement,
VexAttestationVerificationRequest request,
ImmutableDictionary<string, string>.Builder diagnostics)
{
if (statement.Subject is null || statement.Subject.Count != 1)
{
diagnostics["subject.count"] = (statement.Subject?.Count ?? 0).ToString();
return false;
}
var subject = statement.Subject[0];
if (!string.Equals(subject.Name, request.Attestation.ExportId, StringComparison.Ordinal))
{
diagnostics["subject.name"] = subject.Name ?? string.Empty;
return false;
}
if (subject.Digest is null)
{
diagnostics["subject.digest"] = "missing";
return false;
}
var algorithmKey = request.Attestation.Artifact.Algorithm.ToLowerInvariant();
if (!subject.Digest.TryGetValue(algorithmKey, out var digest)
|| !string.Equals(digest, request.Attestation.Artifact.Digest, StringComparison.OrdinalIgnoreCase))
{
diagnostics["subject.digest"] = digest ?? string.Empty;
return false;
}
return true;
}
private bool ValidatePredicate(
VexInTotoStatement statement,
VexAttestationVerificationRequest request,
ImmutableDictionary<string, string>.Builder diagnostics)
{
var predicate = statement.Predicate;
if (predicate is null)
{
diagnostics["predicate.state"] = "missing";
return false;
}
if (!string.Equals(predicate.ExportId, request.Attestation.ExportId, StringComparison.Ordinal))
{
diagnostics["predicate.exportId"] = predicate.ExportId ?? string.Empty;
return false;
}
if (!string.Equals(predicate.QuerySignature, request.Attestation.QuerySignature.Value, StringComparison.Ordinal))
{
diagnostics["predicate.querySignature"] = predicate.QuerySignature ?? string.Empty;
return false;
}
if (!string.Equals(predicate.ArtifactAlgorithm, request.Attestation.Artifact.Algorithm, StringComparison.OrdinalIgnoreCase)
|| !string.Equals(predicate.ArtifactDigest, request.Attestation.Artifact.Digest, StringComparison.OrdinalIgnoreCase))
{
diagnostics["predicate.artifact"] = $"{predicate.ArtifactAlgorithm}:{predicate.ArtifactDigest}";
return false;
}
if (predicate.Format != request.Attestation.Format)
{
diagnostics["predicate.format"] = predicate.Format.ToString();
return false;
}
var createdDelta = (predicate.CreatedAt - request.Attestation.CreatedAt).Duration();
if (createdDelta > _options.MaxClockSkew)
{
diagnostics["predicate.createdAtDelta"] = createdDelta.ToString();
return false;
}
if (!SetEquals(predicate.SourceProviders, request.Attestation.SourceProviders))
{
diagnostics["predicate.sourceProviders"] = string.Join(",", predicate.SourceProviders ?? Array.Empty<string>());
return false;
}
if (request.Attestation.Metadata.Count > 0)
{
if (predicate.Metadata is null)
{
diagnostics["predicate.metadata"] = "missing";
return false;
}
foreach (var kvp in request.Attestation.Metadata)
{
if (!predicate.Metadata.TryGetValue(kvp.Key, out var actual)
|| !string.Equals(actual, kvp.Value, StringComparison.Ordinal))
{
diagnostics[$"predicate.metadata.{kvp.Key}"] = actual ?? string.Empty;
return false;
}
}
}
return true;
}
private bool ValidateMetadataDigest(
DsseEnvelope envelope,
VexAttestationMetadata metadata,
ImmutableDictionary<string, string>.Builder diagnostics)
{
if (string.IsNullOrWhiteSpace(metadata.EnvelopeDigest))
{
diagnostics["metadata.envelopeDigest"] = "missing";
return false;
}
var computed = VexDsseBuilder.ComputeEnvelopeDigest(envelope);
if (!string.Equals(computed, metadata.EnvelopeDigest, StringComparison.OrdinalIgnoreCase))
{
diagnostics["metadata.envelopeDigest"] = metadata.EnvelopeDigest;
diagnostics["metadata.envelopeDigest.computed"] = computed;
return false;
}
diagnostics["metadata.envelopeDigest"] = "match";
return true;
}
private bool ValidateSignedAt(
VexAttestationMetadata metadata,
DateTimeOffset createdAt,
ImmutableDictionary<string, string>.Builder diagnostics)
{
if (metadata.SignedAt is null)
{
diagnostics["metadata.signedAt"] = "missing";
return false;
}
var delta = (metadata.SignedAt.Value - createdAt).Duration();
if (delta > _options.MaxClockSkew)
{
diagnostics["metadata.signedAtDelta"] = delta.ToString();
return false;
}
return true;
}
private async ValueTask<string> VerifyTransparencyAsync(
VexAttestationMetadata metadata,
ImmutableDictionary<string, string>.Builder diagnostics,
CancellationToken cancellationToken)
{
if (metadata.Rekor is null)
{
if (_options.RequireTransparencyLog)
{
diagnostics["rekor.state"] = "missing";
return "missing";
}
diagnostics["rekor.state"] = "disabled";
return "disabled";
}
if (_transparencyLogClient is null)
{
diagnostics["rekor.state"] = "client_unavailable";
return _options.RequireTransparencyLog ? "client_unavailable" : "disabled";
}
try
{
var verified = await _transparencyLogClient.VerifyAsync(metadata.Rekor.Location, cancellationToken).ConfigureAwait(false);
diagnostics["rekor.state"] = verified ? "verified" : "unverified";
return verified ? "verified" : "unverified";
}
catch (Exception ex)
{
diagnostics["rekor.error"] = ex.GetType().Name;
if (_options.AllowOfflineTransparency)
{
diagnostics["rekor.state"] = "offline";
return "offline";
}
diagnostics["rekor.state"] = "unreachable";
return "unreachable";
}
}
private static bool SetEquals(IReadOnlyCollection<string>? left, ImmutableArray<string> right)
{
if (left is null)
{
return right.IsDefaultOrEmpty;
}
if (left.Count != right.Length)
{
return false;
}
var leftSet = new HashSet<string>(left, StringComparer.Ordinal);
return right.All(leftSet.Contains);
}
}

View File

@@ -0,0 +1,111 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Attestation;
public sealed class VexAttestationClientOptions
{
public IReadOnlyDictionary<string, string> DefaultMetadata { get; set; } = ImmutableDictionary<string, string>.Empty;
}
public sealed class VexAttestationClient : IVexAttestationClient
{
private readonly VexDsseBuilder _builder;
private readonly ILogger<VexAttestationClient> _logger;
private readonly TimeProvider _timeProvider;
private readonly IReadOnlyDictionary<string, string> _defaultMetadata;
private readonly ITransparencyLogClient? _transparencyLogClient;
private readonly IVexAttestationVerifier _verifier;
public VexAttestationClient(
VexDsseBuilder builder,
IOptions<VexAttestationClientOptions> options,
ILogger<VexAttestationClient> logger,
IVexAttestationVerifier verifier,
TimeProvider? timeProvider = null,
ITransparencyLogClient? transparencyLogClient = null)
{
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
ArgumentNullException.ThrowIfNull(options);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_verifier = verifier ?? throw new ArgumentNullException(nameof(verifier));
_timeProvider = timeProvider ?? TimeProvider.System;
_defaultMetadata = options.Value.DefaultMetadata;
_transparencyLogClient = transparencyLogClient;
}
public async ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var mergedMetadata = MergeMetadata(request.Metadata, _defaultMetadata);
var envelope = await _builder.CreateEnvelopeAsync(request, mergedMetadata, cancellationToken).ConfigureAwait(false);
var envelopeDigest = VexDsseBuilder.ComputeEnvelopeDigest(envelope);
var signedAt = _timeProvider.GetUtcNow();
var diagnosticsBuilder = ImmutableDictionary<string, string>.Empty
.Add("envelope", JsonSerializer.Serialize(envelope))
.Add("predicateType", "https://stella-ops.org/attestations/vex-export");
VexRekorReference? rekorReference = null;
if (_transparencyLogClient is not null)
{
try
{
var entry = await _transparencyLogClient.SubmitAsync(envelope, cancellationToken).ConfigureAwait(false);
rekorReference = new VexRekorReference("0.2", entry.Location, entry.LogIndex, entry.InclusionProofUrl is not null ? new Uri(entry.InclusionProofUrl) : null);
diagnosticsBuilder = diagnosticsBuilder.Add("rekorLocation", entry.Location);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to submit attestation to Rekor transparency log");
throw;
}
}
var metadata = new VexAttestationMetadata(
predicateType: "https://stella-ops.org/attestations/vex-export",
rekor: rekorReference,
envelopeDigest: envelopeDigest,
signedAt: signedAt);
_logger.LogInformation("Generated DSSE envelope for export {ExportId} ({Digest})", request.ExportId, envelopeDigest);
return new VexAttestationResponse(metadata, diagnosticsBuilder);
}
public ValueTask<VexAttestationVerification> VerifyAsync(
VexAttestationVerificationRequest request,
CancellationToken cancellationToken)
=> _verifier.VerifyAsync(request, cancellationToken);
private static IReadOnlyDictionary<string, string> MergeMetadata(
IReadOnlyDictionary<string, string> requestMetadata,
IReadOnlyDictionary<string, string> defaults)
{
if (defaults.Count == 0)
{
return requestMetadata;
}
var merged = new Dictionary<string, string>(defaults, StringComparer.Ordinal);
foreach (var kvp in requestMetadata)
{
merged[kvp.Key] = kvp.Value;
}
return merged.ToImmutableDictionary(StringComparer.Ordinal);
}
}

View File

@@ -0,0 +1,22 @@
# AGENTS
## Role
Defines shared connector infrastructure for Excititor, including base contexts, result contracts, configuration binding, and helper utilities reused by all connector plug-ins.
## Scope
- `IVexConnector` context implementation, raw store helpers, verification hooks, and telemetry utilities.
- Configuration primitives (YAML parsing, secrets handling guidelines) and options validation.
- Connector lifecycle helpers for retries, paging, `.well-known` discovery, and resume markers.
- Documentation for connector packaging, plugin manifest metadata, and DI registration (see `docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md` and `docs/dev/templates/excititor-connector/`).
## Participants
- All Excititor connector projects reference this module to obtain base classes and context services.
- WebService/Worker instantiate connectors via plugin loader leveraging abstractions defined here.
## Interfaces & contracts
- Connector context, result, and telemetry interfaces; `VexConnectorDescriptor`, `VexConnectorBase`, options binder/validators, authentication helpers.
- Utility classes for HTTP clients, throttling, and deterministic logging.
## In/Out of scope
In: shared abstractions, helper utilities, configuration binding, documentation for connector authors.
Out: provider-specific logic (implemented in individual connector modules), storage persistence, HTTP host code.
## Observability & security expectations
- Provide structured logging helpers, correlation IDs, and metrics instrumentation toggles for connectors.
- Enforce redaction of secrets in logs and config dumps.
## Tests
- Abstraction/unit tests will live in `../StellaOps.Excititor.Connectors.Abstractions.Tests`, covering default behaviors and sample harness.

View File

@@ -0,0 +1,12 @@
using System.Collections.Generic;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Custom validator hook executed after connector options are bound.
/// </summary>
/// <typeparam name="TOptions">Connector-specific options type.</typeparam>
public interface IVexConnectorOptionsValidator<in TOptions>
{
void Validate(VexConnectorDescriptor descriptor, TOptions options, IList<string> errors);
}

View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-ABS-01-001 Connector context & base classes|Team Excititor Connectors|EXCITITOR-CORE-01-003|**DONE (2025-10-17)** Added `StellaOps.Excititor.Connectors.Abstractions` project with `VexConnectorBase`, deterministic logging scopes, metadata builder helpers, and connector descriptors; docs updated to highlight the shared abstractions.|
|EXCITITOR-CONN-ABS-01-002 YAML options & validation|Team Excititor Connectors|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Delivered `VexConnectorOptionsBinder` + binder options/validators, environment-variable expansion, data-annotation checks, and custom validation hooks with documentation updates covering the workflow.|
|EXCITITOR-CONN-ABS-01-003 Plugin packaging & docs|Team Excititor Connectors|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Authored `docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md`, added quick-start template under `docs/dev/templates/excititor-connector/`, and updated module docs to reference the packaging workflow.|

View File

@@ -0,0 +1,99 @@
using System.Collections.Immutable;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Convenience base class for implementing <see cref="IVexConnector" />.
/// </summary>
public abstract class VexConnectorBase : IVexConnector
{
protected VexConnectorBase(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider? timeProvider = null)
{
Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
TimeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
public string Id => Descriptor.Id;
/// <inheritdoc />
public VexProviderKind Kind => Descriptor.Kind;
public VexConnectorDescriptor Descriptor { get; }
protected ILogger Logger { get; }
protected TimeProvider TimeProvider { get; }
protected DateTimeOffset UtcNow() => TimeProvider.GetUtcNow();
protected VexRawDocument CreateRawDocument(
VexDocumentFormat format,
Uri sourceUri,
ReadOnlyMemory<byte> content,
ImmutableDictionary<string, string>? metadata = null)
{
if (sourceUri is null)
{
throw new ArgumentNullException(nameof(sourceUri));
}
var digest = ComputeSha256(content.Span);
var captured = TimeProvider.GetUtcNow();
return new VexRawDocument(
Descriptor.Id,
format,
sourceUri,
captured,
digest,
content,
metadata ?? ImmutableDictionary<string, string>.Empty);
}
protected IDisposable BeginConnectorScope(string operation, IReadOnlyDictionary<string, object?>? metadata = null)
=> VexConnectorLogScope.Begin(Logger, Descriptor, operation, metadata);
protected void LogConnectorEvent(LogLevel level, string eventName, string message, IReadOnlyDictionary<string, object?>? metadata = null, Exception? exception = null)
{
using var scope = BeginConnectorScope(eventName, metadata);
if (exception is null)
{
Logger.Log(level, "{Message}", message);
}
else
{
Logger.Log(level, exception, "{Message}", message);
}
}
protected ImmutableDictionary<string, string> BuildMetadata(Action<VexConnectorMetadataBuilder> configure)
{
ArgumentNullException.ThrowIfNull(configure);
var builder = new VexConnectorMetadataBuilder();
configure(builder);
return builder.Build();
}
private static string ComputeSha256(ReadOnlySpan<byte> content)
{
Span<byte> buffer = stackalloc byte[32];
if (SHA256.TryHashData(content, buffer, out _))
{
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
}
using var sha = SHA256.Create();
var hash = sha.ComputeHash(content.ToArray());
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
public abstract ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken);
public abstract IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken);
public abstract ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,54 @@
using System.Collections.Immutable;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Static descriptor for a Excititor connector plug-in.
/// </summary>
public sealed record VexConnectorDescriptor
{
public VexConnectorDescriptor(string id, VexProviderKind kind, string displayName)
{
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentException("Connector id must be provided.", nameof(id));
}
Id = id;
Kind = kind;
DisplayName = string.IsNullOrWhiteSpace(displayName) ? id : displayName;
}
/// <summary>
/// Stable connector identifier (matches provider id).
/// </summary>
public string Id { get; }
/// <summary>
/// Provider kind served by the connector.
/// </summary>
public VexProviderKind Kind { get; }
/// <summary>
/// Human friendly name used in logs/diagnostics.
/// </summary>
public string DisplayName { get; }
/// <summary>
/// Optional friendly description.
/// </summary>
public string? Description { get; init; }
/// <summary>
/// Document formats the connector is expected to emit.
/// </summary>
public ImmutableArray<VexDocumentFormat> SupportedFormats { get; init; } = ImmutableArray<VexDocumentFormat>.Empty;
/// <summary>
/// Optional tags surfaced in diagnostics (e.g. "beta", "offline").
/// </summary>
public ImmutableArray<string> Tags { get; init; } = ImmutableArray<string>.Empty;
public override string ToString() => $"{Id} ({Kind})";
}

View File

@@ -0,0 +1,50 @@
using System.Linq;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Helper to establish deterministic logging scopes for connector operations.
/// </summary>
public static class VexConnectorLogScope
{
public static IDisposable Begin(ILogger logger, VexConnectorDescriptor descriptor, string operation, IReadOnlyDictionary<string, object?>? metadata = null)
{
ArgumentNullException.ThrowIfNull(logger);
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentException.ThrowIfNullOrEmpty(operation);
var scopeValues = new List<KeyValuePair<string, object?>>
{
new("vex.connector.id", descriptor.Id),
new("vex.connector.kind", descriptor.Kind.ToString()),
new("vex.connector.operation", operation),
};
if (!string.Equals(descriptor.DisplayName, descriptor.Id, StringComparison.Ordinal))
{
scopeValues.Add(new KeyValuePair<string, object?>("vex.connector.displayName", descriptor.DisplayName));
}
if (!string.IsNullOrWhiteSpace(descriptor.Description))
{
scopeValues.Add(new KeyValuePair<string, object?>("vex.connector.description", descriptor.Description));
}
if (!descriptor.Tags.IsDefaultOrEmpty)
{
scopeValues.Add(new KeyValuePair<string, object?>("vex.connector.tags", string.Join(",", descriptor.Tags)));
}
if (metadata is not null)
{
foreach (var kvp in metadata.OrderBy(static pair => pair.Key, StringComparer.Ordinal))
{
scopeValues.Add(new KeyValuePair<string, object?>($"vex.{kvp.Key}", kvp.Value));
}
}
return logger.BeginScope(scopeValues)!;
}
}

View File

@@ -0,0 +1,37 @@
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Builds deterministic metadata dictionaries for raw documents and logging scopes.
/// </summary>
public sealed class VexConnectorMetadataBuilder
{
private readonly SortedDictionary<string, string> _values = new(StringComparer.Ordinal);
public VexConnectorMetadataBuilder Add(string key, string? value)
{
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
{
_values[key] = value!;
}
return this;
}
public VexConnectorMetadataBuilder Add(string key, DateTimeOffset value)
=> Add(key, value.ToUniversalTime().ToString("O"));
public VexConnectorMetadataBuilder AddRange(IEnumerable<KeyValuePair<string, string?>> items)
{
foreach (var item in items)
{
Add(item.Key, item.Value);
}
return this;
}
public ImmutableDictionary<string, string> Build()
=> _values.ToImmutableDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal);
}

View File

@@ -0,0 +1,157 @@
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using Microsoft.Extensions.Configuration;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Provides strongly typed binding and validation for connector options.
/// </summary>
public static class VexConnectorOptionsBinder
{
public static TOptions Bind<TOptions>(
VexConnectorDescriptor descriptor,
VexConnectorSettings settings,
VexConnectorOptionsBinderOptions? options = null,
IEnumerable<IVexConnectorOptionsValidator<TOptions>>? validators = null)
where TOptions : class, new()
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(settings);
var binderSettings = options ?? new VexConnectorOptionsBinderOptions();
var transformed = TransformValues(settings, binderSettings);
var configuration = BuildConfiguration(transformed);
var result = new TOptions();
var errors = new List<string>();
try
{
configuration.Bind(
result,
binderOptions => binderOptions.ErrorOnUnknownConfiguration = !binderSettings.AllowUnknownKeys);
}
catch (InvalidOperationException ex) when (!binderSettings.AllowUnknownKeys)
{
errors.Add(ex.Message);
}
binderSettings.PostConfigure?.Invoke(result);
if (binderSettings.ValidateDataAnnotations)
{
ValidateDataAnnotations(result, errors);
}
if (validators is not null)
{
foreach (var validator in validators)
{
validator?.Validate(descriptor, result, errors);
}
}
if (errors.Count > 0)
{
throw new VexConnectorOptionsValidationException(descriptor.Id, errors);
}
return result;
}
private static ImmutableDictionary<string, string?> TransformValues(
VexConnectorSettings settings,
VexConnectorOptionsBinderOptions binderOptions)
{
var builder = ImmutableDictionary.CreateBuilder<string, string?>(StringComparer.OrdinalIgnoreCase);
foreach (var kvp in settings.Values)
{
var value = kvp.Value;
if (binderOptions.TrimWhitespace && value is not null)
{
value = value.Trim();
}
if (binderOptions.TreatEmptyAsNull && string.IsNullOrEmpty(value))
{
value = null;
}
if (value is not null && binderOptions.ExpandEnvironmentVariables)
{
value = Environment.ExpandEnvironmentVariables(value);
}
if (binderOptions.ValueTransformer is not null)
{
value = binderOptions.ValueTransformer.Invoke(kvp.Key, value);
}
builder[kvp.Key] = value;
}
return builder.ToImmutable();
}
private static IConfiguration BuildConfiguration(ImmutableDictionary<string, string?> values)
{
var sources = new List<KeyValuePair<string, string?>>();
foreach (var kvp in values)
{
if (kvp.Value is not null)
{
sources.Add(new KeyValuePair<string, string?>(kvp.Key, kvp.Value));
}
}
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.Add(new DictionaryConfigurationSource(sources));
return configurationBuilder.Build();
}
private static void ValidateDataAnnotations<TOptions>(TOptions options, IList<string> errors)
{
var validationResults = new List<ValidationResult>();
var validationContext = new ValidationContext(options!);
if (!Validator.TryValidateObject(options!, validationContext, validationResults, validateAllProperties: true))
{
foreach (var validationResult in validationResults)
{
if (!string.IsNullOrWhiteSpace(validationResult.ErrorMessage))
{
errors.Add(validationResult.ErrorMessage);
}
}
}
}
private sealed class DictionaryConfigurationSource : IConfigurationSource
{
private readonly IReadOnlyList<KeyValuePair<string, string?>> _data;
public DictionaryConfigurationSource(IEnumerable<KeyValuePair<string, string?>> data)
{
_data = data?.ToList() ?? new List<KeyValuePair<string, string?>>();
}
public IConfigurationProvider Build(IConfigurationBuilder builder) => new DictionaryConfigurationProvider(_data);
}
private sealed class DictionaryConfigurationProvider : ConfigurationProvider
{
public DictionaryConfigurationProvider(IEnumerable<KeyValuePair<string, string?>> data)
{
foreach (var pair in data)
{
if (pair.Value is not null)
{
Data[pair.Key] = pair.Value;
}
}
}
}
}

View File

@@ -0,0 +1,45 @@
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Customisation options for connector options binding.
/// </summary>
public sealed class VexConnectorOptionsBinderOptions
{
/// <summary>
/// Indicates whether environment variables should be expanded in option values.
/// Defaults to <c>true</c>.
/// </summary>
public bool ExpandEnvironmentVariables { get; set; } = true;
/// <summary>
/// When <c>true</c> the binder trims whitespace around option values.
/// </summary>
public bool TrimWhitespace { get; set; } = true;
/// <summary>
/// Converts empty strings to <c>null</c> before binding. Default: <c>true</c>.
/// </summary>
public bool TreatEmptyAsNull { get; set; } = true;
/// <summary>
/// When <c>false</c>, binding fails if unknown configuration keys are provided.
/// Default: <c>true</c> (permitting unknown keys).
/// </summary>
public bool AllowUnknownKeys { get; set; } = true;
/// <summary>
/// Enables <see cref="System.ComponentModel.DataAnnotations"/> validation after binding.
/// Default: <c>true</c>.
/// </summary>
public bool ValidateDataAnnotations { get; set; } = true;
/// <summary>
/// Optional post-configuration callback executed after binding.
/// </summary>
public Action<object>? PostConfigure { get; set; }
/// <summary>
/// Optional hook to transform raw configuration values before binding.
/// </summary>
public Func<string, string?, string?>? ValueTransformer { get; set; }
}

View File

@@ -0,0 +1,36 @@
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.Abstractions;
public sealed class VexConnectorOptionsValidationException : Exception
{
public VexConnectorOptionsValidationException(
string connectorId,
IEnumerable<string> errors)
: base(BuildMessage(connectorId, errors))
{
ConnectorId = connectorId;
Errors = errors?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
}
public string ConnectorId { get; }
public ImmutableArray<string> Errors { get; }
private static string BuildMessage(string connectorId, IEnumerable<string> errors)
{
var builder = new System.Text.StringBuilder();
builder.Append("Connector options validation failed for '");
builder.Append(connectorId);
builder.Append("'.");
var list = errors?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
if (!list.IsDefaultOrEmpty)
{
builder.Append(" Errors: ");
builder.Append(string.Join("; ", list));
}
return builder.ToString();
}
}

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector responsible for ingesting Cisco CSAF VEX advisories and handing raw documents to normalizers with Cisco-specific metadata.
## Scope
- Discovery of Cisco CSAF collection endpoints, authentication (when required), and pagination routines.
- HTTP retries/backoff, checksum verification, and document deduplication before storage.
- Mapping Cisco advisory identifiers, product hierarchies, and severity hints into connector metadata.
- Surfacing provider trust configuration aligned with policy expectations.
## Participants
- Worker drives scheduled pulls; WebService may trigger manual runs.
- CSAF normalizer consumes raw documents to emit claims.
- Policy module references connector trust hints (e.g., Cisco signing identities).
## Interfaces & contracts
- Implements `IVexConnector` using shared abstractions for HTTP/resume handling.
- Provides options for API tokens, rate limits, and concurrency.
## In/Out of scope
In: data fetching, provider metadata, retry controls, raw document persistence.
Out: normalization/export, attestation, Mongo wiring (handled in other modules).
## Observability & security expectations
- Log fetch batches with document counts/durations; mask credentials.
- Emit metrics for rate-limit hits, retries, and quarantine events.
## Tests
- Unit tests plus HTTP harness fixtures will live in `../StellaOps.Excititor.Connectors.Cisco.CSAF.Tests`.

View File

@@ -0,0 +1,257 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF;
public sealed class CiscoCsafConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:cisco",
kind: VexProviderKind.Vendor,
displayName: "Cisco CSAF")
{
Tags = ImmutableArray.Create("cisco", "csaf"),
};
private readonly CiscoProviderMetadataLoader _metadataLoader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>> _validators;
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
private CiscoConnectorOptions? _options;
private CiscoProviderMetadataResult? _providerMetadata;
public CiscoCsafConnector(
CiscoProviderMetadataLoader metadataLoader,
IHttpClientFactory httpClientFactory,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>>? validators,
ILogger<CiscoCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<CiscoConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary<string, object?>
{
["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length,
["fromOffline"] = _providerMetadata.FromOfflineSnapshot,
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_providerMetadata is null)
{
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
}
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue;
var latestTimestamp = state?.LastUpdated ?? since;
var stateChanged = false;
var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
foreach (var directory in _providerMetadata.Provider.BaseUris)
{
await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false))
{
var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue;
if (published <= since)
{
continue;
}
using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
contentResponse.EnsureSuccessStatusCode();
var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var rawDocument = CreateRawDocument(
VexDocumentFormat.Csaf,
advisory.DocumentUri,
payload,
BuildMetadata(builder => builder
.Add("cisco.csaf.advisoryId", advisory.Id)
.Add("cisco.csaf.revision", advisory.Revision)
.Add("cisco.csaf.published", advisory.Published?.ToString("O"))
.Add("cisco.csaf.modified", advisory.LastModified?.ToString("O"))
.Add("cisco.csaf.sha256", advisory.Sha256)));
if (!digestSet.Add(rawDocument.Digest))
{
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
if (published > latestTimestamp)
{
latestTimestamp = published;
}
yield return rawDocument;
}
}
if (stateChanged)
{
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing.");
private async IAsyncEnumerable<CiscoAdvisoryEntry> EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var nextUri = BuildIndexUri(directory, null);
while (nextUri is not null)
{
using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var page = JsonSerializer.Deserialize<CiscoAdvisoryIndex>(json, _serializerOptions);
if (page?.Advisories is null)
{
yield break;
}
foreach (var advisory in page.Advisories)
{
if (string.IsNullOrWhiteSpace(advisory.Url))
{
continue;
}
if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri))
{
continue;
}
if (!documentUri.IsAbsoluteUri)
{
documentUri = new Uri(directory, documentUri);
}
yield return new CiscoAdvisoryEntry(
advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(),
documentUri,
advisory.Revision,
advisory.Published,
advisory.LastModified,
advisory.Sha256);
}
nextUri = ResolveNextUri(directory, page.Next);
}
}
private static Uri BuildIndexUri(Uri directory, string? relative)
{
if (string.IsNullOrWhiteSpace(relative))
{
var baseText = directory.ToString();
if (!baseText.EndsWith('/'))
{
baseText += "/";
}
return new Uri(new Uri(baseText, UriKind.Absolute), "index.json");
}
if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute))
{
return absolute;
}
var baseTextRelative = directory.ToString();
if (!baseTextRelative.EndsWith('/'))
{
baseTextRelative += "/";
}
return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative);
}
private static Uri? ResolveNextUri(Uri directory, string? next)
{
if (string.IsNullOrWhiteSpace(next))
{
return null;
}
return BuildIndexUri(directory, next);
}
private sealed record CiscoAdvisoryIndex
{
public List<CiscoAdvisory>? Advisories { get; init; }
public string? Next { get; init; }
}
private sealed record CiscoAdvisory
{
public string? Id { get; init; }
public string? Url { get; init; }
public string? Revision { get; init; }
public DateTimeOffset? Published { get; init; }
public DateTimeOffset? LastModified { get; init; }
public string? Sha256 { get; init; }
}
private sealed record CiscoAdvisoryEntry(
string Id,
Uri DocumentUri,
string? Revision,
DateTimeOffset? Published,
DateTimeOffset? LastModified,
string? Sha256);
}

View File

@@ -0,0 +1,58 @@
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
public sealed class CiscoConnectorOptions : IValidatableObject
{
public const string HttpClientName = "cisco-csaf";
/// <summary>
/// Endpoint for Cisco CSAF provider metadata discovery.
/// </summary>
[Required]
public string MetadataUri { get; set; } = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json";
/// <summary>
/// Optional bearer token used when Cisco endpoints require authentication.
/// </summary>
public string? ApiToken { get; set; }
/// <summary>
/// How long provider metadata remains cached.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6);
/// <summary>
/// Whether to prefer offline snapshots when fetching metadata.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// When set, provider metadata will be persisted to the given file path.
/// </summary>
public bool PersistOfflineSnapshot { get; set; }
public string? OfflineSnapshotPath { get; set; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (string.IsNullOrWhiteSpace(MetadataUri))
{
yield return new ValidationResult("MetadataUri must be provided.", new[] { nameof(MetadataUri) });
}
else if (!Uri.TryCreate(MetadataUri, UriKind.Absolute, out _))
{
yield return new ValidationResult("MetadataUri must be an absolute URI.", new[] { nameof(MetadataUri) });
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
yield return new ValidationResult("MetadataCacheDuration must be greater than zero.", new[] { nameof(MetadataCacheDuration) });
}
if (PersistOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
yield return new ValidationResult("OfflineSnapshotPath must be provided when PersistOfflineSnapshot is enabled.", new[] { nameof(OfflineSnapshotPath) });
}
}
}

View File

@@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
public sealed class CiscoConnectorOptionsValidator : IVexConnectorOptionsValidator<CiscoConnectorOptions>
{
public void Validate(VexConnectorDescriptor descriptor, CiscoConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
var validationResults = new List<ValidationResult>();
if (!Validator.TryValidateObject(options, new ValidationContext(options), validationResults, validateAllProperties: true))
{
foreach (var result in validationResults)
{
errors.Add(result.ErrorMessage ?? "Cisco connector options validation failed.");
}
}
}
}

View File

@@ -0,0 +1,52 @@
using System.ComponentModel.DataAnnotations;
using System.Net.Http.Headers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection;
public static class CiscoConnectorServiceCollectionExtensions
{
public static IServiceCollection AddCiscoCsafConnector(this IServiceCollection services, Action<CiscoConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.AddOptions<CiscoConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
})
.PostConfigure(options =>
{
Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true);
});
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.TryAddEnumerable(ServiceDescriptor.Singleton<IVexConnectorOptionsValidator<CiscoConnectorOptions>, CiscoConnectorOptionsValidator>());
services.AddHttpClient(CiscoConnectorOptions.HttpClientName)
.ConfigureHttpClient((provider, client) =>
{
var options = provider.GetRequiredService<IOptions<CiscoConnectorOptions>>().Value;
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
if (!string.IsNullOrWhiteSpace(options.ApiToken))
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiToken);
}
});
services.AddSingleton<CiscoProviderMetadataLoader>();
services.AddSingleton<IVexConnector, CiscoCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,332 @@
using System.Collections.Immutable;
using System.Net;
using System.Net.Http.Headers;
using System.Text.Json;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
public sealed class CiscoProviderMetadataLoader
{
public const string CacheKey = "StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly ILogger<CiscoProviderMetadataLoader> _logger;
private readonly CiscoConnectorOptions _options;
private readonly IFileSystem _fileSystem;
private readonly JsonSerializerOptions _serializerOptions;
private readonly SemaphoreSlim _semaphore = new(1, 1);
public CiscoProviderMetadataLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IOptions<CiscoConnectorOptions> options,
ILogger<CiscoProviderMetadataLoader> logger,
IFileSystem? fileSystem = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value ?? throw new ArgumentNullException(nameof(options));
_fileSystem = fileSystem ?? new FileSystem();
_serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
}
public async Task<CiscoProviderMetadataResult> LoadAsync(CancellationToken cancellationToken)
{
if (_memoryCache.TryGetValue<CacheEntry>(CacheKey, out var cached) && cached is not null && !cached.IsExpired())
{
_logger.LogDebug("Returning cached Cisco provider metadata (expires {Expires}).", cached.ExpiresAt);
return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_memoryCache.TryGetValue<CacheEntry>(CacheKey, out cached) && cached is not null && !cached.IsExpired())
{
return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true);
}
CacheEntry? previous = cached;
if (!_options.PreferOfflineSnapshot)
{
var network = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false);
if (network is not null)
{
StoreCache(network);
return new CiscoProviderMetadataResult(network.Provider, network.FetchedAt, false, false);
}
}
var offline = TryLoadFromOffline();
if (offline is not null)
{
var entry = offline with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
FromOffline = true,
};
StoreCache(entry);
return new CiscoProviderMetadataResult(entry.Provider, entry.FetchedAt, true, false);
}
throw new InvalidOperationException("Unable to load Cisco CSAF provider metadata from network or offline snapshot.");
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri);
if (!string.IsNullOrWhiteSpace(_options.ApiToken))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiToken);
}
if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag))
{
request.Headers.IfNoneMatch.Add(etag);
}
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotModified && previous is not null)
{
_logger.LogDebug("Cisco provider metadata not modified (etag {ETag}).", previous.ETag);
return previous with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
};
}
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var provider = ParseProvider(payload);
var etagHeader = response.Headers.ETag?.ToString();
if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
try
{
_fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
}
return new CacheEntry(
provider,
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
etagHeader,
FromOffline: false);
}
catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot)
{
_logger.LogWarning(ex, "Failed to fetch Cisco provider metadata from {Uri}; falling back to offline snapshot when available.", _options.MetadataUri);
return null;
}
}
private CacheEntry? TryLoadFromOffline()
{
if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath))
{
_logger.LogWarning("Cisco offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath);
var provider = ParseProvider(payload);
return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, null, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Cisco provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath);
return null;
}
}
private VexProvider ParseProvider(string payload)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Cisco provider metadata payload was empty.");
}
ProviderMetadataDocument? document;
try
{
document = JsonSerializer.Deserialize<ProviderMetadataDocument>(payload, _serializerOptions);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Failed to parse Cisco provider metadata.", ex);
}
if (document?.Metadata?.Publisher?.ContactDetails is null || string.IsNullOrWhiteSpace(document.Metadata.Publisher.ContactDetails.Id))
{
throw new InvalidOperationException("Cisco provider metadata did not include a publisher identifier.");
}
var discovery = new VexProviderDiscovery(document.Discovery?.WellKnown, document.Discovery?.RolIe);
var trust = document.Trust is null
? VexProviderTrust.Default
: new VexProviderTrust(
document.Trust.Weight ?? 1.0,
document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty),
document.Trust.PgpFingerprints ?? Enumerable.Empty<string>());
var directories = document.Distributions?.Directories is null
? Enumerable.Empty<Uri>()
: document.Distributions.Directories
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Select(static s => Uri.TryCreate(s, UriKind.Absolute, out var uri) ? uri : null)
.Where(static uri => uri is not null)!
.Select(static uri => uri!);
return new VexProvider(
id: document.Metadata.Publisher.ContactDetails.Id,
displayName: document.Metadata.Publisher.Name ?? document.Metadata.Publisher.ContactDetails.Id,
kind: document.Metadata.Publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub,
baseUris: directories,
discovery: discovery,
trust: trust,
enabled: true);
}
private void StoreCache(CacheEntry entry)
{
var options = new MemoryCacheEntryOptions
{
AbsoluteExpiration = entry.ExpiresAt,
};
_memoryCache.Set(CacheKey, entry, options);
}
private sealed record CacheEntry(
VexProvider Provider,
DateTimeOffset FetchedAt,
DateTimeOffset ExpiresAt,
string? ETag,
bool FromOffline)
{
public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt;
}
}
public sealed record CiscoProviderMetadataResult(
VexProvider Provider,
DateTimeOffset FetchedAt,
bool FromOfflineSnapshot,
bool ServedFromCache);
#region document models
internal sealed class ProviderMetadataDocument
{
[System.Text.Json.Serialization.JsonPropertyName("metadata")]
public ProviderMetadataMetadata Metadata { get; set; } = new();
[System.Text.Json.Serialization.JsonPropertyName("discovery")]
public ProviderMetadataDiscovery? Discovery { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("trust")]
public ProviderMetadataTrust? Trust { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("distributions")]
public ProviderMetadataDistributions? Distributions { get; set; }
}
internal sealed class ProviderMetadataMetadata
{
[System.Text.Json.Serialization.JsonPropertyName("publisher")]
public ProviderMetadataPublisher Publisher { get; set; } = new();
}
internal sealed class ProviderMetadataPublisher
{
[System.Text.Json.Serialization.JsonPropertyName("name")]
public string? Name { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("category")]
public string? Category { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("contact_details")]
public ProviderMetadataPublisherContact ContactDetails { get; set; } = new();
}
internal sealed class ProviderMetadataPublisherContact
{
[System.Text.Json.Serialization.JsonPropertyName("id")]
public string? Id { get; set; }
}
internal sealed class ProviderMetadataDiscovery
{
[System.Text.Json.Serialization.JsonPropertyName("well_known")]
public Uri? WellKnown { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("rolie")]
public Uri? RolIe { get; set; }
}
internal sealed class ProviderMetadataTrust
{
[System.Text.Json.Serialization.JsonPropertyName("weight")]
public double? Weight { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("cosign")]
public ProviderMetadataTrustCosign? Cosign { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("pgp_fingerprints")]
public string[]? PgpFingerprints { get; set; }
}
internal sealed class ProviderMetadataTrustCosign
{
[System.Text.Json.Serialization.JsonPropertyName("issuer")]
public string? Issuer { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("identity_pattern")]
public string? IdentityPattern { get; set; }
}
internal sealed class ProviderMetadataDistributions
{
[System.Text.Json.Serialization.JsonPropertyName("directories")]
public string[]? Directories { get; set; }
}
#endregion

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-CISCO-01-001 Endpoint discovery & auth plumbing|Team Excititor Connectors Cisco|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added `CiscoProviderMetadataLoader` with bearer token support, offline snapshot fallback, DI helpers, and tests covering network/offline discovery to unblock subsequent fetch work.|
|EXCITITOR-CONN-CISCO-01-002 CSAF pull loop & pagination|Team Excititor Connectors Cisco|EXCITITOR-CONN-CISCO-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** Implemented paginated advisory fetch using provider directories, raw document persistence with dedupe/state tracking, offline resiliency, and unit coverage.|
|EXCITITOR-CONN-CISCO-01-003 Provider trust metadata|Team Excititor Connectors Cisco|EXCITITOR-CONN-CISCO-01-002, EXCITITOR-POLICY-01-001|**DOING (2025-10-19)** Prereqs confirmed (both DONE); implementing cosign/PGP trust metadata emission and advisory provenance hints for policy weighting.|

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector for Microsoft Security Response Center (MSRC) CSAF advisories, handling authenticated downloads, throttling, and raw document persistence.
## Scope
- MSRC API onboarding (AAD client credentials), metadata discovery, and CSAF listing retrieval.
- Download pipeline with retry/backoff, checksum validation, and document deduplication.
- Mapping MSRC-specific identifiers (CVE, ADV, KB) and remediation guidance into connector metadata.
- Emitting trust metadata (AAD issuer, signing certificates) for policy weighting.
## Participants
- Worker schedules MSRC pulls honoring rate limits; WebService may trigger manual runs for urgent updates.
- CSAF normalizer processes retrieved documents into claims.
- Policy subsystem references connector trust hints for consensus scoring.
## Interfaces & contracts
- Implements `IVexConnector`, requires configuration options for tenant/client/secret or managed identity.
- Uses shared HTTP helpers, resume markers, and telemetry from Abstractions module.
## In/Out of scope
In: authenticated fetching, raw document storage, metadata mapping, retry logic.
Out: normalization/export, attestation, storage implementations (handled elsewhere).
## Observability & security expectations
- Log request batches, rate-limit responses, and token refresh events without leaking secrets.
- Track metrics for documents fetched, retries, and failure categories.
## Tests
- Connector tests with mocked MSRC endpoints and AAD token flow will live in `../StellaOps.Excititor.Connectors.MSRC.CSAF.Tests`.

View File

@@ -0,0 +1,185 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
public interface IMsrcTokenProvider
{
ValueTask<MsrcAccessToken> GetAccessTokenAsync(CancellationToken cancellationToken);
}
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
{
private const string CachePrefix = "StellaOps.Excititor.Connectors.MSRC.CSAF.Token";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<MsrcTokenProvider> _logger;
private readonly TimeProvider _timeProvider;
private readonly MsrcConnectorOptions _options;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
public MsrcTokenProvider(
IHttpClientFactory httpClientFactory,
IMemoryCache cache,
IFileSystem fileSystem,
IOptions<MsrcConnectorOptions> options,
ILogger<MsrcTokenProvider> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate(_fileSystem);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async ValueTask<MsrcAccessToken> GetAccessTokenAsync(CancellationToken cancellationToken)
{
if (_options.PreferOfflineToken)
{
return LoadOfflineToken();
}
var cacheKey = CreateCacheKey();
if (_cache.TryGetValue<MsrcAccessToken>(cacheKey, out var cachedToken) &&
cachedToken is not null &&
!cachedToken.IsExpired(_timeProvider.GetUtcNow()))
{
return cachedToken;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_cache.TryGetValue<MsrcAccessToken>(cacheKey, out cachedToken) &&
cachedToken is not null &&
!cachedToken.IsExpired(_timeProvider.GetUtcNow()))
{
return cachedToken;
}
var token = await RequestTokenAsync(cancellationToken).ConfigureAwait(false);
var absoluteExpiration = token.ExpiresAt == DateTimeOffset.MaxValue
? (DateTimeOffset?)null
: token.ExpiresAt;
var options = new MemoryCacheEntryOptions();
if (absoluteExpiration.HasValue)
{
options.AbsoluteExpiration = absoluteExpiration.Value;
}
_cache.Set(cacheKey, token, options);
return token;
}
finally
{
_refreshLock.Release();
}
}
private MsrcAccessToken LoadOfflineToken()
{
if (!string.IsNullOrWhiteSpace(_options.StaticAccessToken))
{
return new MsrcAccessToken(_options.StaticAccessToken!, "Bearer", DateTimeOffset.MaxValue);
}
if (string.IsNullOrWhiteSpace(_options.OfflineTokenPath))
{
throw new InvalidOperationException("Offline token mode is enabled but no token was provided.");
}
if (!_fileSystem.File.Exists(_options.OfflineTokenPath))
{
throw new InvalidOperationException($"Offline token path '{_options.OfflineTokenPath}' does not exist.");
}
var token = _fileSystem.File.ReadAllText(_options.OfflineTokenPath).Trim();
if (string.IsNullOrEmpty(token))
{
throw new InvalidOperationException("Offline token file was empty.");
}
return new MsrcAccessToken(token, "Bearer", DateTimeOffset.MaxValue);
}
private async Task<MsrcAccessToken> RequestTokenAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Fetching MSRC AAD access token for tenant {TenantId}.", _options.TenantId);
var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.TokenClientName);
using var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
{
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["client_id"] = _options.ClientId,
["client_secret"] = _options.ClientSecret!,
["grant_type"] = "client_credentials",
["scope"] = string.IsNullOrWhiteSpace(_options.Scope) ? MsrcConnectorOptions.DefaultScope : _options.Scope,
}),
};
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException($"Failed to acquire MSRC access token ({(int)response.StatusCode}). Response: {payload}");
}
var tokenResponse = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("Token endpoint returned an empty payload.");
if (string.IsNullOrWhiteSpace(tokenResponse.AccessToken))
{
throw new InvalidOperationException("Token endpoint response did not include an access_token.");
}
var now = _timeProvider.GetUtcNow();
var expiresAt = tokenResponse.ExpiresIn > _options.ExpiryLeewaySeconds
? now.AddSeconds(tokenResponse.ExpiresIn - _options.ExpiryLeewaySeconds)
: now.AddMinutes(5);
return new MsrcAccessToken(tokenResponse.AccessToken!, tokenResponse.TokenType ?? "Bearer", expiresAt);
}
private string CreateCacheKey()
=> $"{CachePrefix}:{_options.TenantId}:{_options.ClientId}:{_options.Scope}";
private Uri BuildTokenUri()
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
public void Dispose() => _refreshLock.Dispose();
private sealed record TokenResponse
{
[JsonPropertyName("access_token")]
public string? AccessToken { get; init; }
[JsonPropertyName("token_type")]
public string? TokenType { get; init; }
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; init; }
}
}
public sealed record MsrcAccessToken(string Value, string Type, DateTimeOffset ExpiresAt)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}

View File

@@ -0,0 +1,211 @@
using System;
using System.Globalization;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
public sealed class MsrcConnectorOptions
{
public const string TokenClientName = "excititor.connector.msrc.token";
public const string DefaultScope = "https://api.msrc.microsoft.com/.default";
public const string ApiClientName = "excititor.connector.msrc.api";
public const string DefaultBaseUri = "https://api.msrc.microsoft.com/sug/v2.0/";
public const string DefaultLocale = "en-US";
public const string DefaultApiVersion = "2024-08-01";
/// <summary>
/// Azure AD tenant identifier (GUID or domain).
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application (client) identifier.
/// </summary>
public string ClientId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application secret for client credential flow.
/// </summary>
public string? ClientSecret { get; set; }
/// <summary>
/// OAuth scope requested for MSRC API access.
/// </summary>
public string Scope { get; set; } = DefaultScope;
/// <summary>
/// When true, token acquisition is skipped and the connector expects offline handling.
/// </summary>
public bool PreferOfflineToken { get; set; }
/// <summary>
/// Optional path to a pre-provisioned bearer token used when <see cref="PreferOfflineToken"/> is enabled.
/// </summary>
public string? OfflineTokenPath { get; set; }
/// <summary>
/// Optional fixed bearer token for constrained environments (e.g., short-lived offline bundles).
/// </summary>
public string? StaticAccessToken { get; set; }
/// <summary>
/// Minimum buffer (seconds) subtracted from token expiry before refresh.
/// </summary>
public int ExpiryLeewaySeconds { get; set; } = 60;
/// <summary>
/// Base URI for MSRC Security Update Guide API.
/// </summary>
public Uri BaseUri { get; set; } = new(DefaultBaseUri, UriKind.Absolute);
/// <summary>
/// Locale requested when fetching summaries.
/// </summary>
public string Locale { get; set; } = DefaultLocale;
/// <summary>
/// API version appended to MSRC requests.
/// </summary>
public string ApiVersion { get; set; } = DefaultApiVersion;
/// <summary>
/// Page size used while enumerating summaries.
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// Maximum CSAF advisories fetched per connector run.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 200;
/// <summary>
/// Overlap window applied when resuming from the last modified cursor.
/// </summary>
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
/// <summary>
/// Delay between CSAF downloads to respect rate limits.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Maximum retry attempts for summary/detail fetch operations.
/// </summary>
public int MaxRetryAttempts { get; set; } = 3;
/// <summary>
/// Base delay applied between retries (jitter handled by connector).
/// </summary>
public TimeSpan RetryBaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Optional lower bound for initial synchronisation when no cursor is stored.
/// </summary>
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
/// <summary>
/// Maximum number of document digests persisted for deduplication.
/// </summary>
public int MaxTrackedDigests { get; set; } = 2048;
public void Validate(IFileSystem? fileSystem = null)
{
if (PreferOfflineToken)
{
if (string.IsNullOrWhiteSpace(OfflineTokenPath) && string.IsNullOrWhiteSpace(StaticAccessToken))
{
throw new InvalidOperationException("OfflineTokenPath or StaticAccessToken must be provided when PreferOfflineToken is enabled.");
}
}
else
{
if (string.IsNullOrWhiteSpace(TenantId))
{
throw new InvalidOperationException("TenantId is required when not operating in offline token mode.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("ClientId is required when not operating in offline token mode.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("ClientSecret is required when not operating in offline token mode.");
}
}
if (string.IsNullOrWhiteSpace(Scope))
{
Scope = DefaultScope;
}
if (ExpiryLeewaySeconds < 10)
{
ExpiryLeewaySeconds = 10;
}
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("BaseUri must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(Locale))
{
throw new InvalidOperationException("Locale must be provided.");
}
if (!CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
}
if (string.IsNullOrWhiteSpace(ApiVersion))
{
throw new InvalidOperationException("ApiVersion must be provided.");
}
if (PageSize <= 0 || PageSize > 500)
{
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
{
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
}
if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10))
{
throw new InvalidOperationException($"{nameof(RequestDelay)} must be between 0 and 10 seconds.");
}
if (MaxRetryAttempts <= 0 || MaxRetryAttempts > 10)
{
throw new InvalidOperationException($"{nameof(MaxRetryAttempts)} must be between 1 and 10.");
}
if (RetryBaseDelay < TimeSpan.Zero || RetryBaseDelay > TimeSpan.FromMinutes(5))
{
throw new InvalidOperationException($"{nameof(RetryBaseDelay)} must be between 0 and 5 minutes.");
}
if (MaxTrackedDigests <= 0 || MaxTrackedDigests > 10000)
{
throw new InvalidOperationException($"{nameof(MaxTrackedDigests)} must be between 1 and 10000.");
}
if (!string.IsNullOrWhiteSpace(OfflineTokenPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineTokenPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,58 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using System.IO.Abstractions;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.DependencyInjection;
public static class MsrcConnectorServiceCollectionExtensions
{
public static IServiceCollection AddMsrcCsafConnector(this IServiceCollection services, Action<MsrcConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<MsrcConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddHttpClient(MsrcConnectorOptions.TokenClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddHttpClient(MsrcConnectorOptions.ApiClientName)
.ConfigureHttpClient((provider, client) =>
{
var options = provider.GetRequiredService<IOptions<MsrcConnectorOptions>>().Value;
client.BaseAddress = options.BaseUri;
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<IMsrcTokenProvider, MsrcTokenProvider>();
services.AddSingleton<IVexConnector, MsrcCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,581 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF;
public sealed class MsrcCsafConnector : VexConnectorBase
{
private const string QuarantineMetadataKey = "excititor.quarantine.reason";
private const string FormatMetadataKey = "msrc.csaf.format";
private const string VulnerabilityMetadataKey = "msrc.vulnerabilityId";
private const string AdvisoryIdMetadataKey = "msrc.advisoryId";
private const string LastModifiedMetadataKey = "msrc.lastModified";
private const string ReleaseDateMetadataKey = "msrc.releaseDate";
private const string CvssSeverityMetadataKey = "msrc.severity";
private const string CvrfUrlMetadataKey = "msrc.cvrfUrl";
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:msrc",
kind: VexProviderKind.Vendor,
displayName: "Microsoft MSRC CSAF")
{
Description = "Authenticated connector for Microsoft Security Response Center CSAF advisories.",
SupportedFormats = ImmutableArray.Create(VexDocumentFormat.Csaf),
Tags = ImmutableArray.Create("microsoft", "csaf", "vendor"),
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMsrcTokenProvider _tokenProvider;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IOptions<MsrcConnectorOptions> _options;
private readonly ILogger<MsrcCsafConnector> _logger;
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
private MsrcConnectorOptions? _validatedOptions;
public MsrcCsafConnector(
IHttpClientFactory httpClientFactory,
IMsrcTokenProvider tokenProvider,
IVexConnectorStateRepository stateRepository,
IOptions<MsrcConnectorOptions> options,
ILogger<MsrcCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered.");
options.Validate();
_validatedOptions = options;
LogConnectorEvent(
LogLevel.Information,
"validate",
"Validated MSRC CSAF connector options.",
new Dictionary<string, object?>
{
["baseUri"] = options.BaseUri.ToString(),
["locale"] = options.Locale,
["apiVersion"] = options.ApiVersion,
["pageSize"] = options.PageSize,
["maxAdvisories"] = options.MaxAdvisoriesPerFetch,
});
return ValueTask.CompletedTask;
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(
VexConnectorContext context,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var options = EnsureOptionsValidated();
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var (from, to) = CalculateWindow(context.Since, state, options);
LogConnectorEvent(
LogLevel.Information,
"fetch.window",
$"Fetching MSRC CSAF advisories updated between {from:O} and {to:O}.",
new Dictionary<string, object?>
{
["from"] = from,
["to"] = to,
["cursorOverlapSeconds"] = options.CursorOverlap.TotalSeconds,
});
var client = await CreateAuthenticatedClientAsync(options, cancellationToken).ConfigureAwait(false);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var latest = state?.LastUpdated ?? from;
var fetched = 0;
var stateChanged = false;
await foreach (var summary in EnumerateSummariesAsync(client, options, from, to, cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
if (fetched >= options.MaxAdvisoriesPerFetch)
{
break;
}
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
LogConnectorEvent(LogLevel.Debug, "skip.no-cvrf", $"Skipping MSRC advisory {summary.Id} because no CSAF URL was provided.");
continue;
}
var documentUri = ResolveCvrfUri(options.BaseUri, summary.CvrfUrl);
VexRawDocument? rawDocument = null;
try
{
rawDocument = await DownloadCsafAsync(client, summary, documentUri, options, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogConnectorEvent(LogLevel.Warning, "fetch.error", $"Failed to download MSRC CSAF package {documentUri}.", new Dictionary<string, object?>
{
["advisoryId"] = summary.Id,
["vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
}, ex);
await Task.Delay(GetRetryDelay(options, 1), cancellationToken).ConfigureAwait(false);
continue;
}
if (!digestSet.Add(rawDocument.Digest))
{
LogConnectorEvent(LogLevel.Debug, "skip.duplicate", $"Skipping MSRC CSAF package {documentUri} because it was already processed.");
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
fetched++;
latest = DetermineLatest(summary, latest) ?? latest;
var quarantineReason = rawDocument.Metadata.TryGetValue(QuarantineMetadataKey, out var reason) ? reason : null;
if (quarantineReason is not null)
{
LogConnectorEvent(LogLevel.Warning, "quarantine", $"Quarantined MSRC CSAF package {documentUri} ({quarantineReason}).");
continue;
}
yield return rawDocument;
if (options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
if (stateChanged)
{
if (digestList.Count > options.MaxTrackedDigests)
{
var trimmed = digestList.Count - options.MaxTrackedDigests;
digestList.RemoveRange(0, trimmed);
}
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latest == DateTimeOffset.MinValue ? state?.LastUpdated : latest,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
LogConnectorEvent(
LogLevel.Information,
"fetch.completed",
$"MSRC CSAF fetch completed with {fetched} new documents.",
new Dictionary<string, object?>
{
["fetched"] = fetched,
["stateChanged"] = stateChanged,
["lastUpdated"] = latest,
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("MSRC CSAF connector relies on CSAF normalizers for document processing.");
private async Task<VexRawDocument> DownloadCsafAsync(
HttpClient client,
MsrcVulnerabilitySummary summary,
Uri documentUri,
MsrcConnectorOptions options,
CancellationToken cancellationToken)
{
using var response = await SendWithRetryAsync(
client,
() => new HttpRequestMessage(HttpMethod.Get, documentUri),
options,
cancellationToken).ConfigureAwait(false);
var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var validation = ValidateCsafPayload(payload);
var metadata = BuildMetadata(builder =>
{
builder.Add(AdvisoryIdMetadataKey, summary.Id);
builder.Add(VulnerabilityMetadataKey, summary.VulnerabilityId ?? summary.Id);
builder.Add(CvrfUrlMetadataKey, documentUri.ToString());
builder.Add(FormatMetadataKey, validation.Format);
if (!string.IsNullOrWhiteSpace(summary.Severity))
{
builder.Add(CvssSeverityMetadataKey, summary.Severity);
}
if (summary.LastModifiedDate is not null)
{
builder.Add(LastModifiedMetadataKey, summary.LastModifiedDate.Value.ToString("O"));
}
if (summary.ReleaseDate is not null)
{
builder.Add(ReleaseDateMetadataKey, summary.ReleaseDate.Value.ToString("O"));
}
if (!string.IsNullOrWhiteSpace(validation.QuarantineReason))
{
builder.Add(QuarantineMetadataKey, validation.QuarantineReason);
}
if (response.Headers.ETag is not null)
{
builder.Add("http.etag", response.Headers.ETag.Tag);
}
if (response.Content.Headers.LastModified is { } lastModified)
{
builder.Add("http.lastModified", lastModified.ToString("O"));
}
});
return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, payload, metadata);
}
private async Task<HttpClient> CreateAuthenticatedClientAsync(MsrcConnectorOptions options, CancellationToken cancellationToken)
{
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.ApiClientName);
client.DefaultRequestHeaders.Remove("Authorization");
client.DefaultRequestHeaders.Add("Authorization", $"{token.Type} {token.Value}");
client.DefaultRequestHeaders.Remove("Accept-Language");
client.DefaultRequestHeaders.Add("Accept-Language", options.Locale);
client.DefaultRequestHeaders.Remove("api-version");
client.DefaultRequestHeaders.Add("api-version", options.ApiVersion);
client.DefaultRequestHeaders.Remove("Accept");
client.DefaultRequestHeaders.Add("Accept", "application/json");
return client;
}
private async Task<HttpResponseMessage> SendWithRetryAsync(
HttpClient client,
Func<HttpRequestMessage> requestFactory,
MsrcConnectorOptions options,
CancellationToken cancellationToken)
{
Exception? lastError = null;
HttpResponseMessage? response = null;
for (var attempt = 1; attempt <= options.MaxRetryAttempts; attempt++)
{
response?.Dispose();
using var request = requestFactory();
try
{
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return response;
}
if (!ShouldRetry(response.StatusCode) || attempt == options.MaxRetryAttempts)
{
response.EnsureSuccessStatusCode();
}
}
catch (Exception ex) when (IsTransient(ex) && attempt < options.MaxRetryAttempts)
{
lastError = ex;
LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex);
}
catch (Exception)
{
response?.Dispose();
throw;
}
await Task.Delay(GetRetryDelay(options, attempt), cancellationToken).ConfigureAwait(false);
}
response?.Dispose();
throw lastError ?? new InvalidOperationException("MSRC request retries exhausted.");
}
private TimeSpan GetRetryDelay(MsrcConnectorOptions options, int attempt)
{
var baseDelay = options.RetryBaseDelay.TotalMilliseconds;
var multiplier = Math.Pow(2, Math.Max(0, attempt - 1));
var jitter = Random.Shared.NextDouble() * baseDelay * 0.25;
var delayMs = Math.Min(baseDelay * multiplier + jitter, TimeSpan.FromMinutes(5).TotalMilliseconds);
return TimeSpan.FromMilliseconds(delayMs);
}
private async IAsyncEnumerable<MsrcVulnerabilitySummary> EnumerateSummariesAsync(
HttpClient client,
MsrcConnectorOptions options,
DateTimeOffset from,
DateTimeOffset to,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var fetched = 0;
var requestUri = BuildSummaryUri(options, from, to);
while (requestUri is not null && fetched < options.MaxAdvisoriesPerFetch)
{
using var response = await SendWithRetryAsync(
client,
() => new HttpRequestMessage(HttpMethod.Get, requestUri),
options,
cancellationToken).ConfigureAwait(false);
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var payload = await JsonSerializer.DeserializeAsync<MsrcSummaryResponse>(stream, _serializerOptions, cancellationToken).ConfigureAwait(false)
?? new MsrcSummaryResponse();
foreach (var summary in payload.Value)
{
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
continue;
}
yield return summary;
fetched++;
if (fetched >= options.MaxAdvisoriesPerFetch)
{
yield break;
}
}
if (string.IsNullOrWhiteSpace(payload.NextLink))
{
break;
}
if (!Uri.TryCreate(payload.NextLink, UriKind.Absolute, out requestUri))
{
LogConnectorEvent(LogLevel.Warning, "pagination.invalid", $"MSRC pagination returned invalid next link '{payload.NextLink}'.");
break;
}
}
}
private static Uri BuildSummaryUri(MsrcConnectorOptions options, DateTimeOffset from, DateTimeOffset to)
{
var baseText = options.BaseUri.ToString().TrimEnd('/');
var builder = new StringBuilder(baseText.Length + 128);
builder.Append(baseText);
if (!baseText.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase))
{
builder.Append("/vulnerabilities");
}
builder.Append("?");
builder.Append("$top=").Append(options.PageSize);
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(from.ToUniversalTime().ToString("O")));
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(to.ToUniversalTime().ToString("O")));
builder.Append("&$orderby=lastModifiedDate");
builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale));
builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion));
return new Uri(builder.ToString(), UriKind.Absolute);
}
private (DateTimeOffset From, DateTimeOffset To) CalculateWindow(
DateTimeOffset? contextSince,
VexConnectorState? state,
MsrcConnectorOptions options)
{
var now = UtcNow();
var since = contextSince ?? state?.LastUpdated ?? options.InitialLastModified ?? now.AddDays(-30);
if (state?.LastUpdated is { } persisted && persisted > since)
{
since = persisted;
}
if (options.CursorOverlap > TimeSpan.Zero)
{
since = since.Add(-options.CursorOverlap);
}
if (since < now.AddYears(-20))
{
since = now.AddYears(-20);
}
return (since, now);
}
private static bool ShouldRetry(HttpStatusCode statusCode)
=> statusCode == HttpStatusCode.TooManyRequests ||
(int)statusCode >= 500;
private static bool IsTransient(Exception exception)
=> exception is HttpRequestException or IOException or TaskCanceledException;
private static Uri ResolveCvrfUri(Uri baseUri, string cvrfUrl)
=> Uri.TryCreate(cvrfUrl, UriKind.Absolute, out var absolute)
? absolute
: new Uri(baseUri, cvrfUrl);
private static CsafValidationResult ValidateCsafPayload(ReadOnlyMemory<byte> payload)
{
try
{
if (IsZip(payload.Span))
{
using var zipStream = new MemoryStream(payload.ToArray(), writable: false);
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true);
var entry = archive.Entries.FirstOrDefault(e => e.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
?? archive.Entries.FirstOrDefault();
if (entry is null)
{
return new CsafValidationResult("zip", "Zip archive did not contain any entries.");
}
using var entryStream = entry.Open();
using var reader = new StreamReader(entryStream, Encoding.UTF8);
using var json = JsonDocument.Parse(reader.ReadToEnd());
return CsafValidationResult.Valid("zip");
}
if (IsGzip(payload.Span))
{
using var input = new MemoryStream(payload.ToArray(), writable: false);
using var gzip = new GZipStream(input, CompressionMode.Decompress);
using var reader = new StreamReader(gzip, Encoding.UTF8);
using var json = JsonDocument.Parse(reader.ReadToEnd());
return CsafValidationResult.Valid("gzip");
}
using var jsonDocument = JsonDocument.Parse(payload);
return CsafValidationResult.Valid("json");
}
catch (JsonException ex)
{
return new CsafValidationResult("json", $"JSON parse failed: {ex.Message}");
}
catch (InvalidDataException ex)
{
return new CsafValidationResult("invalid", ex.Message);
}
catch (EndOfStreamException ex)
{
return new CsafValidationResult("invalid", ex.Message);
}
}
private static bool IsZip(ReadOnlySpan<byte> content)
=> content.Length > 3 && content[0] == 0x50 && content[1] == 0x4B;
private static bool IsGzip(ReadOnlySpan<byte> content)
=> content.Length > 2 && content[0] == 0x1F && content[1] == 0x8B;
private static DateTimeOffset? DetermineLatest(MsrcVulnerabilitySummary summary, DateTimeOffset? current)
{
var candidate = summary.LastModifiedDate ?? summary.ReleaseDate;
if (candidate is null)
{
return current;
}
if (current is null || candidate > current)
{
return candidate;
}
return current;
}
private MsrcConnectorOptions EnsureOptionsValidated()
{
if (_validatedOptions is not null)
{
return _validatedOptions;
}
var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered.");
options.Validate();
_validatedOptions = options;
return options;
}
private sealed record CsafValidationResult(string Format, string? QuarantineReason)
{
public static CsafValidationResult Valid(string format) => new(format, null);
}
}
internal sealed record MsrcSummaryResponse
{
[JsonPropertyName("value")]
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
[JsonPropertyName("@odata.nextLink")]
public string? NextLink { get; init; }
}
internal sealed record MsrcVulnerabilitySummary
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string? VulnerabilityId { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-MS-01-001 AAD onboarding & token cache|Team Excititor Connectors MSRC|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added MSRC connector project with configurable AAD options, token provider (offline/online modes), DI wiring, and unit tests covering caching and fallback scenarios.|
|EXCITITOR-CONN-MS-01-002 CSAF download pipeline|Team Excititor Connectors MSRC|EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** Prereqs verified (EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003); drafting fetch/retry plan and storage wiring before implementation of CSAF package download, checksum validation, and quarantine flows.|
|EXCITITOR-CONN-MS-01-003 Trust metadata & provenance hints|Team Excititor Connectors MSRC|EXCITITOR-CONN-MS-01-002, EXCITITOR-POLICY-01-001|TODO Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration.|

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector for OCI registry OpenVEX attestations, discovering images, downloading attestations, and projecting statements into raw storage.
## Scope
- OCI registry discovery, authentication (cosign OIDC/key), and ref resolution for provided image digests/tags.
- Fetching DSSE envelopes, verifying signatures (delegated to Attestation module), and persisting raw statements.
- Mapping OCI manifest metadata (repository, digest, subject) to connector provenance.
- Managing offline bundles that seed attestations without registry access.
## Participants
- Worker schedules polls for configured registries/images; WebService supports manual refresh.
- OpenVEX normalizer consumes statements to create claims.
- Attestation module is reused to verify upstream envelopes prior to storage.
## Interfaces & contracts
- Implements `IVexConnector` with options for image list, auth, parallelism, and offline file seeds.
- Utilizes shared abstractions for retries, telemetry, and resume markers.
## In/Out of scope
In: OCI interaction, attestation retrieval, verification trigger, raw persistence.
Out: normalization/export, policy evaluation, storage implementation.
## Observability & security expectations
- Log image references, attestation counts, verification outcomes; redact credentials.
- Emit metrics for attestation reuse ratio, verification duration, and failures.
## Tests
- Connector tests with mock OCI registry/attestation responses will live in `../StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests`.

View File

@@ -0,0 +1,110 @@
using System;
using System.IO.Abstractions;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
public sealed record CosignKeylessIdentity(
string Issuer,
string Subject,
Uri? FulcioUrl,
Uri? RekorUrl,
string? ClientId,
string? ClientSecret,
string? Audience,
string? IdentityToken);
public sealed record CosignKeyPairIdentity(
string PrivateKeyPath,
string? Password,
string? CertificatePath,
Uri? RekorUrl,
string? FulcioRootPath);
public sealed record OciCosignAuthority(
CosignCredentialMode Mode,
CosignKeylessIdentity? Keyless,
CosignKeyPairIdentity? KeyPair,
bool RequireSignature,
TimeSpan VerifyTimeout);
public static class OciCosignAuthorityFactory
{
public static OciCosignAuthority Create(OciCosignVerificationOptions options, IFileSystem? fileSystem = null)
{
ArgumentNullException.ThrowIfNull(options);
CosignKeylessIdentity? keyless = null;
CosignKeyPairIdentity? keyPair = null;
switch (options.Mode)
{
case CosignCredentialMode.None:
break;
case CosignCredentialMode.Keyless:
keyless = CreateKeyless(options.Keyless);
break;
case CosignCredentialMode.KeyPair:
keyPair = CreateKeyPair(options.KeyPair, fileSystem);
break;
default:
throw new InvalidOperationException($"Unsupported Cosign credential mode '{options.Mode}'.");
}
return new OciCosignAuthority(
Mode: options.Mode,
Keyless: keyless,
KeyPair: keyPair,
RequireSignature: options.RequireSignature,
VerifyTimeout: options.VerifyTimeout);
}
private static CosignKeylessIdentity CreateKeyless(CosignKeylessOptions options)
{
ArgumentNullException.ThrowIfNull(options);
Uri? fulcio = null;
Uri? rekor = null;
if (!string.IsNullOrWhiteSpace(options.FulcioUrl))
{
fulcio = new Uri(options.FulcioUrl, UriKind.Absolute);
}
if (!string.IsNullOrWhiteSpace(options.RekorUrl))
{
rekor = new Uri(options.RekorUrl, UriKind.Absolute);
}
return new CosignKeylessIdentity(
Issuer: options.Issuer!,
Subject: options.Subject!,
FulcioUrl: fulcio,
RekorUrl: rekor,
ClientId: options.ClientId,
ClientSecret: options.ClientSecret,
Audience: options.Audience,
IdentityToken: options.IdentityToken);
}
private static CosignKeyPairIdentity CreateKeyPair(CosignKeyPairOptions options, IFileSystem? fileSystem)
{
ArgumentNullException.ThrowIfNull(options);
Uri? rekor = null;
if (!string.IsNullOrWhiteSpace(options.RekorUrl))
{
rekor = new Uri(options.RekorUrl, UriKind.Absolute);
}
return new CosignKeyPairIdentity(
PrivateKeyPath: options.PrivateKeyPath!,
Password: options.Password,
CertificatePath: options.CertificatePath,
RekorUrl: rekor,
FulcioRootPath: options.FulcioRootPath);
}
}

View File

@@ -0,0 +1,59 @@
using System;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
public enum OciRegistryAuthMode
{
Anonymous = 0,
Basic = 1,
IdentityToken = 2,
RefreshToken = 3,
}
public sealed record OciRegistryAuthorization(
string? RegistryAuthority,
OciRegistryAuthMode Mode,
string? Username,
string? Password,
string? IdentityToken,
string? RefreshToken,
bool AllowAnonymousFallback)
{
public static OciRegistryAuthorization Create(OciRegistryAuthenticationOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var mode = OciRegistryAuthMode.Anonymous;
string? username = null;
string? password = null;
string? identityToken = null;
string? refreshToken = null;
if (!string.IsNullOrWhiteSpace(options.IdentityToken))
{
mode = OciRegistryAuthMode.IdentityToken;
identityToken = options.IdentityToken;
}
else if (!string.IsNullOrWhiteSpace(options.RefreshToken))
{
mode = OciRegistryAuthMode.RefreshToken;
refreshToken = options.RefreshToken;
}
else if (!string.IsNullOrWhiteSpace(options.Username))
{
mode = OciRegistryAuthMode.Basic;
username = options.Username;
password = options.Password;
}
return new OciRegistryAuthorization(
RegistryAuthority: options.RegistryAuthority,
Mode: mode,
Username: username,
Password: password,
IdentityToken: identityToken,
RefreshToken: refreshToken,
AllowAnonymousFallback: options.AllowAnonymousFallback);
}
}

View File

@@ -0,0 +1,321 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
public sealed class OciOpenVexAttestationConnectorOptions
{
public const string HttpClientName = "excititor.connector.oci.openvex.attest";
public IList<OciImageSubscriptionOptions> Images { get; } = new List<OciImageSubscriptionOptions>();
public OciRegistryAuthenticationOptions Registry { get; } = new();
public OciCosignVerificationOptions Cosign { get; } = new();
public OciOfflineBundleOptions Offline { get; } = new();
public TimeSpan DiscoveryCacheDuration { get; set; } = TimeSpan.FromMinutes(15);
public int MaxParallelResolutions { get; set; } = 4;
public bool AllowHttpRegistries { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (Images.Count == 0)
{
throw new InvalidOperationException("At least one OCI image reference must be configured.");
}
foreach (var image in Images)
{
image.Validate();
}
if (MaxParallelResolutions <= 0 || MaxParallelResolutions > 32)
{
throw new InvalidOperationException("MaxParallelResolutions must be between 1 and 32.");
}
if (DiscoveryCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("DiscoveryCacheDuration must be a positive time span.");
}
Registry.Validate();
Cosign.Validate(fileSystem);
Offline.Validate(fileSystem);
if (!AllowHttpRegistries && Images.Any(i => i.Reference is not null && i.Reference.StartsWith("http://", StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException("HTTP (non-TLS) registries are disabled. Enable AllowHttpRegistries to permit them.");
}
}
}
public sealed class OciImageSubscriptionOptions
{
private OciImageReference? _parsedReference;
/// <summary>
/// Gets or sets the OCI reference (e.g. registry.example.com/repository:tag or registry.example.com/repository@sha256:abcdef).
/// </summary>
public string? Reference { get; set; }
/// <summary>
/// Optional friendly name used in logs when referencing this subscription.
/// </summary>
public string? DisplayName { get; set; }
/// <summary>
/// Optional file path for an offline attestation bundle associated with this image.
/// </summary>
public string? OfflineBundlePath { get; set; }
/// <summary>
/// Optional override for the expected subject digest. When provided, discovery will verify resolved digests match.
/// </summary>
public string? ExpectedSubjectDigest { get; set; }
internal OciImageReference? ParsedReference => _parsedReference;
public void Validate()
{
if (string.IsNullOrWhiteSpace(Reference))
{
throw new InvalidOperationException("Image Reference is required for OCI OpenVEX attestation connector.");
}
_parsedReference = OciImageReferenceParser.Parse(Reference);
if (!string.IsNullOrWhiteSpace(ExpectedSubjectDigest))
{
if (!ExpectedSubjectDigest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("ExpectedSubjectDigest must start with 'sha256:'.");
}
if (ExpectedSubjectDigest.Length != "sha256:".Length + 64)
{
throw new InvalidOperationException("ExpectedSubjectDigest must contain a 64-character hexadecimal hash.");
}
}
}
}
public sealed class OciRegistryAuthenticationOptions
{
/// <summary>
/// Optional registry authority filter (e.g. registry.example.com:5000). When set it must match image references.
/// </summary>
public string? RegistryAuthority { get; set; }
public string? Username { get; set; }
public string? Password { get; set; }
public string? IdentityToken { get; set; }
public string? RefreshToken { get; set; }
public bool AllowAnonymousFallback { get; set; } = true;
public void Validate()
{
var hasUser = !string.IsNullOrWhiteSpace(Username);
var hasPassword = !string.IsNullOrWhiteSpace(Password);
var hasIdentityToken = !string.IsNullOrWhiteSpace(IdentityToken);
var hasRefreshToken = !string.IsNullOrWhiteSpace(RefreshToken);
if (hasIdentityToken && (hasUser || hasPassword))
{
throw new InvalidOperationException("IdentityToken cannot be combined with Username/Password for OCI registry authentication.");
}
if (hasRefreshToken && (hasUser || hasPassword))
{
throw new InvalidOperationException("RefreshToken cannot be combined with Username/Password for OCI registry authentication.");
}
if (hasUser != hasPassword)
{
throw new InvalidOperationException("Username and Password must be provided together for OCI registry authentication.");
}
if (!string.IsNullOrWhiteSpace(RegistryAuthority) && RegistryAuthority.Contains('/', StringComparison.Ordinal))
{
throw new InvalidOperationException("RegistryAuthority must not contain path segments.");
}
}
}
public sealed class OciCosignVerificationOptions
{
public CosignCredentialMode Mode { get; set; } = CosignCredentialMode.Keyless;
public CosignKeylessOptions Keyless { get; } = new();
public CosignKeyPairOptions KeyPair { get; } = new();
public bool RequireSignature { get; set; } = true;
public TimeSpan VerifyTimeout { get; set; } = TimeSpan.FromSeconds(30);
public void Validate(IFileSystem? fileSystem = null)
{
if (VerifyTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("VerifyTimeout must be a positive time span.");
}
switch (Mode)
{
case CosignCredentialMode.None:
break;
case CosignCredentialMode.Keyless:
Keyless.Validate();
break;
case CosignCredentialMode.KeyPair:
KeyPair.Validate(fileSystem);
break;
default:
throw new InvalidOperationException($"Unsupported Cosign credential mode '{Mode}'.");
}
}
}
public enum CosignCredentialMode
{
None = 0,
Keyless = 1,
KeyPair = 2,
}
public sealed class CosignKeylessOptions
{
public string? Issuer { get; set; }
public string? Subject { get; set; }
public string? FulcioUrl { get; set; } = "https://fulcio.sigstore.dev";
public string? RekorUrl { get; set; } = "https://rekor.sigstore.dev";
public string? ClientId { get; set; }
public string? ClientSecret { get; set; }
public string? Audience { get; set; }
public string? IdentityToken { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Issuer))
{
throw new InvalidOperationException("Cosign keyless Issuer must be provided.");
}
if (string.IsNullOrWhiteSpace(Subject))
{
throw new InvalidOperationException("Cosign keyless Subject must be provided.");
}
if (!string.IsNullOrWhiteSpace(FulcioUrl) && !Uri.TryCreate(FulcioUrl, UriKind.Absolute, out var fulcio))
{
throw new InvalidOperationException("FulcioUrl must be an absolute URI when provided.");
}
if (!string.IsNullOrWhiteSpace(RekorUrl) && !Uri.TryCreate(RekorUrl, UriKind.Absolute, out var rekor))
{
throw new InvalidOperationException("RekorUrl must be an absolute URI when provided.");
}
if (!string.IsNullOrWhiteSpace(ClientSecret) && string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("Cosign keyless ClientId must be provided when ClientSecret is specified.");
}
}
}
public sealed class CosignKeyPairOptions
{
public string? PrivateKeyPath { get; set; }
public string? Password { get; set; }
public string? CertificatePath { get; set; }
public string? RekorUrl { get; set; }
public string? FulcioRootPath { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (string.IsNullOrWhiteSpace(PrivateKeyPath))
{
throw new InvalidOperationException("PrivateKeyPath must be provided for Cosign key pair mode.");
}
var fs = fileSystem ?? new FileSystem();
if (!fs.File.Exists(PrivateKeyPath))
{
throw new InvalidOperationException($"Cosign private key file not found: {PrivateKeyPath}");
}
if (!string.IsNullOrWhiteSpace(CertificatePath) && !fs.File.Exists(CertificatePath))
{
throw new InvalidOperationException($"Cosign certificate file not found: {CertificatePath}");
}
if (!string.IsNullOrWhiteSpace(FulcioRootPath) && !fs.File.Exists(FulcioRootPath))
{
throw new InvalidOperationException($"Cosign Fulcio root file not found: {FulcioRootPath}");
}
if (!string.IsNullOrWhiteSpace(RekorUrl) && !Uri.TryCreate(RekorUrl, UriKind.Absolute, out _))
{
throw new InvalidOperationException("RekorUrl must be an absolute URI when provided for Cosign key pair mode.");
}
}
}
public sealed class OciOfflineBundleOptions
{
public string? RootDirectory { get; set; }
public bool PreferOffline { get; set; }
public bool AllowNetworkFallback { get; set; } = true;
public string? DefaultBundleFileName { get; set; } = "openvex-attestations.tgz";
public bool RequireBundles { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (string.IsNullOrWhiteSpace(RootDirectory))
{
return;
}
var fs = fileSystem ?? new FileSystem();
if (!fs.Directory.Exists(RootDirectory))
{
if (PreferOffline || RequireBundles)
{
throw new InvalidOperationException($"Offline bundle root directory '{RootDirectory}' does not exist.");
}
fs.Directory.CreateDirectory(RootDirectory);
}
}
}

View File

@@ -0,0 +1,35 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
public sealed class OciOpenVexAttestationConnectorOptionsValidator : IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public OciOpenVexAttestationConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(
VexConnectorDescriptor descriptor,
OciOpenVexAttestationConnectorOptions options,
IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,52 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection;
public static class OciOpenVexAttestationConnectorServiceCollectionExtensions
{
public static IServiceCollection AddOciOpenVexAttestationConnector(
this IServiceCollection services,
Action<OciOpenVexAttestationConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<OciOpenVexAttestationConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
});
services.AddSingleton<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>, OciOpenVexAttestationConnectorOptionsValidator>();
services.AddSingleton<OciAttestationDiscoveryService>();
services.AddSingleton<OciAttestationFetcher>();
services.AddSingleton<IVexConnector, OciOpenVexAttestationConnector>();
services.AddHttpClient(OciOpenVexAttestationConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.cncf.openvex.v1+json");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
return services;
}
}

View File

@@ -0,0 +1,11 @@
using System.Collections.Immutable;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciAttestationDiscoveryResult(
ImmutableArray<OciAttestationTarget> Targets,
OciRegistryAuthorization RegistryAuthorization,
OciCosignAuthority CosignAuthority,
bool PreferOffline,
bool AllowNetworkFallback);

View File

@@ -0,0 +1,188 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed class OciAttestationDiscoveryService
{
private readonly IMemoryCache _memoryCache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<OciAttestationDiscoveryService> _logger;
public OciAttestationDiscoveryService(
IMemoryCache memoryCache,
IFileSystem fileSystem,
ILogger<OciAttestationDiscoveryService> logger)
{
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public Task<OciAttestationDiscoveryResult> LoadAsync(
OciOpenVexAttestationConnectorOptions options,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue(cacheKey, out OciAttestationDiscoveryResult? cached) && cached is not null)
{
_logger.LogDebug("Using cached OCI attestation discovery result for {ImageCount} images.", cached.Targets.Length);
return Task.FromResult(cached);
}
var targets = new List<OciAttestationTarget>(options.Images.Count);
foreach (var image in options.Images)
{
cancellationToken.ThrowIfCancellationRequested();
var parsed = image.ParsedReference ?? OciImageReferenceParser.Parse(image.Reference!);
var offlinePath = ResolveOfflinePath(options, image, parsed);
OciOfflineBundleReference? offline = null;
if (!string.IsNullOrWhiteSpace(offlinePath))
{
var fullPath = _fileSystem.Path.GetFullPath(offlinePath!);
var exists = _fileSystem.File.Exists(fullPath) || _fileSystem.Directory.Exists(fullPath);
if (!exists && options.Offline.RequireBundles)
{
throw new InvalidOperationException($"Required offline bundle '{fullPath}' for reference '{parsed.Canonical}' was not found.");
}
offline = new OciOfflineBundleReference(fullPath, exists, image.ExpectedSubjectDigest);
}
targets.Add(new OciAttestationTarget(parsed, image.ExpectedSubjectDigest, offline));
}
var authorization = OciRegistryAuthorization.Create(options.Registry);
var cosignAuthority = OciCosignAuthorityFactory.Create(options.Cosign, _fileSystem);
var result = new OciAttestationDiscoveryResult(
targets.ToImmutableArray(),
authorization,
cosignAuthority,
options.Offline.PreferOffline,
options.Offline.AllowNetworkFallback);
_memoryCache.Set(cacheKey, result, options.DiscoveryCacheDuration);
return Task.FromResult(result);
}
private string? ResolveOfflinePath(
OciOpenVexAttestationConnectorOptions options,
OciImageSubscriptionOptions image,
OciImageReference parsed)
{
if (!string.IsNullOrWhiteSpace(image.OfflineBundlePath))
{
return image.OfflineBundlePath;
}
if (string.IsNullOrWhiteSpace(options.Offline.RootDirectory))
{
return null;
}
var root = options.Offline.RootDirectory!;
var segments = new List<string> { SanitizeSegment(parsed.Registry) };
var repositoryParts = parsed.Repository.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (repositoryParts.Length == 0)
{
segments.Add(SanitizeSegment(parsed.Repository));
}
else
{
foreach (var part in repositoryParts)
{
segments.Add(SanitizeSegment(part));
}
}
var versionSegment = parsed.Digest is not null
? SanitizeSegment(parsed.Digest)
: SanitizeSegment(parsed.Tag ?? "latest");
segments.Add(versionSegment);
var combined = _fileSystem.Path.Combine(new[] { root }.Concat(segments).ToArray());
if (!string.IsNullOrWhiteSpace(options.Offline.DefaultBundleFileName))
{
combined = _fileSystem.Path.Combine(combined, options.Offline.DefaultBundleFileName!);
}
return combined;
}
private static string SanitizeSegment(string value)
{
if (string.IsNullOrEmpty(value))
{
return "_";
}
var builder = new StringBuilder(value.Length);
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch) || ch is '-' or '_' or '.')
{
builder.Append(ch);
}
else
{
builder.Append('_');
}
}
return builder.Length == 0 ? "_" : builder.ToString();
}
private static string CreateCacheKey(OciOpenVexAttestationConnectorOptions options)
{
using var sha = SHA256.Create();
var builder = new StringBuilder();
builder.AppendLine("oci-openvex-attest");
builder.AppendLine(options.MaxParallelResolutions.ToString());
builder.AppendLine(options.AllowHttpRegistries.ToString());
builder.AppendLine(options.Offline.PreferOffline.ToString());
builder.AppendLine(options.Offline.AllowNetworkFallback.ToString());
foreach (var image in options.Images)
{
builder.AppendLine(image.Reference ?? string.Empty);
builder.AppendLine(image.ExpectedSubjectDigest ?? string.Empty);
builder.AppendLine(image.OfflineBundlePath ?? string.Empty);
}
if (!string.IsNullOrWhiteSpace(options.Offline.RootDirectory))
{
builder.AppendLine(options.Offline.RootDirectory);
builder.AppendLine(options.Offline.DefaultBundleFileName ?? string.Empty);
}
builder.AppendLine(options.Registry.RegistryAuthority ?? string.Empty);
builder.AppendLine(options.Registry.AllowAnonymousFallback.ToString());
var bytes = Encoding.UTF8.GetBytes(builder.ToString());
var hashBytes = sha.ComputeHash(bytes);
return Convert.ToHexString(hashBytes);
}
}

View File

@@ -0,0 +1,6 @@
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciAttestationTarget(
OciImageReference Image,
string? ExpectedSubjectDigest,
OciOfflineBundleReference? OfflineBundle);

View File

@@ -0,0 +1,27 @@
using System;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciImageReference(string Registry, string Repository, string? Tag, string? Digest, string OriginalReference, string Scheme = "https")
{
public string Canonical =>
Digest is not null
? $"{Registry}/{Repository}@{Digest}"
: Tag is not null
? $"{Registry}/{Repository}:{Tag}"
: $"{Registry}/{Repository}";
public bool HasDigest => !string.IsNullOrWhiteSpace(Digest);
public bool HasTag => !string.IsNullOrWhiteSpace(Tag);
public OciImageReference WithDigest(string digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
throw new ArgumentException("Digest must be provided.", nameof(digest));
}
return this with { Digest = digest };
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Text.RegularExpressions;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
internal static class OciImageReferenceParser
{
private static readonly Regex DigestRegex = new(@"^(?<algorithm>[A-Za-z0-9+._-]+):(?<hash>[A-Fa-f0-9]{32,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex RepositoryRegex = new(@"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
public static OciImageReference Parse(string reference)
{
if (string.IsNullOrWhiteSpace(reference))
{
throw new InvalidOperationException("OCI reference cannot be empty.");
}
var trimmed = reference.Trim();
string original = trimmed;
var scheme = "https";
if (trimmed.StartsWith("oci://", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed.Substring("oci://".Length);
}
if (trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed.Substring("https://".Length);
scheme = "https";
}
else if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed.Substring("http://".Length);
scheme = "http";
}
var firstSlash = trimmed.IndexOf('/');
if (firstSlash <= 0)
{
throw new InvalidOperationException($"OCI reference '{reference}' must include a registry and repository component.");
}
var registry = trimmed[..firstSlash];
var remainder = trimmed[(firstSlash + 1)..];
if (!LooksLikeRegistry(registry))
{
throw new InvalidOperationException($"OCI reference '{reference}' is missing an explicit registry component.");
}
string? digest = null;
string? tag = null;
var digestIndex = remainder.IndexOf('@');
if (digestIndex >= 0)
{
digest = remainder[(digestIndex + 1)..];
remainder = remainder[..digestIndex];
if (!DigestRegex.IsMatch(digest))
{
throw new InvalidOperationException($"Digest segment '{digest}' is not a valid OCI digest.");
}
}
var tagIndex = remainder.LastIndexOf(':');
if (tagIndex >= 0)
{
tag = remainder[(tagIndex + 1)..];
remainder = remainder[..tagIndex];
if (string.IsNullOrWhiteSpace(tag))
{
throw new InvalidOperationException("OCI tag segment cannot be empty.");
}
if (tag.Contains('/', StringComparison.Ordinal))
{
throw new InvalidOperationException("OCI tag segment cannot contain '/'.");
}
}
var repository = remainder;
if (string.IsNullOrWhiteSpace(repository))
{
throw new InvalidOperationException("OCI repository segment cannot be empty.");
}
if (!RepositoryRegex.IsMatch(repository))
{
throw new InvalidOperationException($"Repository segment '{repository}' is not valid per OCI distribution rules.");
}
return new OciImageReference(
Registry: registry,
Repository: repository,
Tag: tag,
Digest: digest,
OriginalReference: original,
Scheme: scheme);
}
private static bool LooksLikeRegistry(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (value.Equals("localhost", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (value.Contains('.', StringComparison.Ordinal) || value.Contains(':', StringComparison.Ordinal))
{
return true;
}
// IPv4/IPv6 simplified check
if (value.Length >= 3 && char.IsDigit(value[0]))
{
return true;
}
return false;
}
}

View File

@@ -0,0 +1,5 @@
using System;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciOfflineBundleReference(string Path, bool Exists, string? ExpectedSubjectDigest);

View File

@@ -0,0 +1,14 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
internal sealed record OciArtifactDescriptor(
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("artifactType")] string? ArtifactType,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations);
internal sealed record OciReferrerIndex(
[property: JsonPropertyName("referrers")] IReadOnlyList<OciArtifactDescriptor> Referrers);

View File

@@ -0,0 +1,13 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
public sealed record OciAttestationDocument(
Uri SourceUri,
ReadOnlyMemory<byte> Content,
ImmutableDictionary<string, string> Metadata,
string? SubjectDigest,
string? ArtifactDigest,
string? ArtifactType,
string SourceKind);

View File

@@ -0,0 +1,258 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Abstractions;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Net.Http;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using System.Formats.Tar;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
public sealed class OciAttestationFetcher
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly IFileSystem _fileSystem;
private readonly ILogger<OciAttestationFetcher> _logger;
public OciAttestationFetcher(
IHttpClientFactory httpClientFactory,
IFileSystem fileSystem,
ILogger<OciAttestationFetcher> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async IAsyncEnumerable<OciAttestationDocument> FetchAsync(
OciAttestationDiscoveryResult discovery,
OciOpenVexAttestationConnectorOptions options,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(discovery);
ArgumentNullException.ThrowIfNull(options);
foreach (var target in discovery.Targets)
{
cancellationToken.ThrowIfCancellationRequested();
bool yieldedOffline = false;
if (target.OfflineBundle is not null && target.OfflineBundle.Exists)
{
await foreach (var offlineDocument in ReadOfflineAsync(target, cancellationToken))
{
yieldedOffline = true;
yield return offlineDocument;
}
if (!discovery.AllowNetworkFallback)
{
continue;
}
}
if (discovery.PreferOffline && yieldedOffline && !discovery.AllowNetworkFallback)
{
continue;
}
if (!discovery.PreferOffline || discovery.AllowNetworkFallback || !yieldedOffline)
{
await foreach (var registryDocument in FetchFromRegistryAsync(discovery, options, target, cancellationToken))
{
yield return registryDocument;
}
}
}
}
private async IAsyncEnumerable<OciAttestationDocument> ReadOfflineAsync(
OciAttestationTarget target,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var offline = target.OfflineBundle!;
var path = _fileSystem.Path.GetFullPath(offline.Path);
if (!_fileSystem.File.Exists(path))
{
if (offline.Exists)
{
_logger.LogWarning("Offline bundle {Path} disappeared before processing.", path);
}
yield break;
}
var extension = _fileSystem.Path.GetExtension(path).ToLowerInvariant();
var subjectDigest = target.Image.Digest ?? target.ExpectedSubjectDigest;
if (string.Equals(extension, ".json", StringComparison.OrdinalIgnoreCase) ||
string.Equals(extension, ".dsse", StringComparison.OrdinalIgnoreCase))
{
var bytes = await _fileSystem.File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
var metadata = BuildOfflineMetadata(target, path, entryName: null, subjectDigest);
yield return new OciAttestationDocument(
new Uri(path, UriKind.Absolute),
bytes,
metadata,
subjectDigest,
null,
null,
"offline");
yield break;
}
if (string.Equals(extension, ".tgz", StringComparison.OrdinalIgnoreCase) ||
string.Equals(extension, ".gz", StringComparison.OrdinalIgnoreCase) ||
string.Equals(extension, ".tar", StringComparison.OrdinalIgnoreCase))
{
await foreach (var document in ReadTarArchiveAsync(target, path, subjectDigest, cancellationToken))
{
yield return document;
}
yield break;
}
// Default: treat as binary blob.
var fallbackBytes = await _fileSystem.File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
var fallbackMetadata = BuildOfflineMetadata(target, path, entryName: null, subjectDigest);
yield return new OciAttestationDocument(
new Uri(path, UriKind.Absolute),
fallbackBytes,
fallbackMetadata,
subjectDigest,
null,
null,
"offline");
}
private async IAsyncEnumerable<OciAttestationDocument> ReadTarArchiveAsync(
OciAttestationTarget target,
string path,
string? subjectDigest,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await using var fileStream = _fileSystem.File.OpenRead(path);
Stream archiveStream = fileStream;
if (path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase) ||
path.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase))
{
archiveStream = new GZipStream(fileStream, CompressionMode.Decompress, leaveOpen: false);
}
using var tarReader = new TarReader(archiveStream, leaveOpen: false);
TarEntry? entry;
while ((entry = await tarReader.GetNextEntryAsync(copyData: false, cancellationToken).ConfigureAwait(false)) is not null)
{
if (entry.EntryType is not TarEntryType.RegularFile || entry.DataStream is null)
{
continue;
}
await using var entryStream = entry.DataStream;
using var buffer = new MemoryStream();
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
var metadata = BuildOfflineMetadata(target, path, entry.Name, subjectDigest);
var sourceUri = new Uri($"{_fileSystem.Path.GetFullPath(path)}#{entry.Name}", UriKind.Absolute);
yield return new OciAttestationDocument(
sourceUri,
buffer.ToArray(),
metadata,
subjectDigest,
null,
null,
"offline");
}
}
private async IAsyncEnumerable<OciAttestationDocument> FetchFromRegistryAsync(
OciAttestationDiscoveryResult discovery,
OciOpenVexAttestationConnectorOptions options,
OciAttestationTarget target,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var registryClient = new OciRegistryClient(
_httpClientFactory,
_logger,
discovery.RegistryAuthorization,
options);
var subjectDigest = target.Image.Digest ?? target.ExpectedSubjectDigest;
if (string.IsNullOrWhiteSpace(subjectDigest))
{
subjectDigest = await registryClient.ResolveDigestAsync(target.Image, cancellationToken).ConfigureAwait(false);
}
if (string.IsNullOrWhiteSpace(subjectDigest))
{
_logger.LogWarning("Unable to resolve subject digest for {Reference}; skipping registry fetch.", target.Image.Canonical);
yield break;
}
if (!string.IsNullOrWhiteSpace(target.ExpectedSubjectDigest) &&
!string.Equals(target.ExpectedSubjectDigest, subjectDigest, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Resolved digest {Resolved} does not match expected digest {Expected} for {Reference}.",
subjectDigest,
target.ExpectedSubjectDigest,
target.Image.Canonical);
}
var descriptors = await registryClient.ListReferrersAsync(target.Image, subjectDigest, cancellationToken).ConfigureAwait(false);
if (descriptors.Count == 0)
{
yield break;
}
foreach (var descriptor in descriptors)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await registryClient.DownloadAttestationAsync(target.Image, descriptor, subjectDigest, cancellationToken).ConfigureAwait(false);
if (document is not null)
{
yield return document;
}
}
}
private static ImmutableDictionary<string, string> BuildOfflineMetadata(
OciAttestationTarget target,
string bundlePath,
string? entryName,
string? subjectDigest)
{
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
builder["oci.image.registry"] = target.Image.Registry;
builder["oci.image.repository"] = target.Image.Repository;
builder["oci.image.reference"] = target.Image.Canonical;
if (!string.IsNullOrWhiteSpace(subjectDigest))
{
builder["oci.image.subjectDigest"] = subjectDigest;
}
if (!string.IsNullOrWhiteSpace(target.ExpectedSubjectDigest))
{
builder["oci.image.expectedSubjectDigest"] = target.ExpectedSubjectDigest!;
}
builder["oci.attestation.sourceKind"] = "offline";
builder["oci.attestation.source"] = bundlePath;
if (!string.IsNullOrWhiteSpace(entryName))
{
builder["oci.attestation.bundleEntry"] = entryName!;
}
return builder.ToImmutable();
}
}

View File

@@ -0,0 +1,362 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
internal sealed class OciRegistryClient
{
private const string ManifestMediaType = "application/vnd.oci.image.manifest.v1+json";
private const string ReferrersArtifactType = "application/vnd.dsse.envelope.v1+json";
private const string DsseMediaType = "application/vnd.dsse.envelope.v1+json";
private const string OpenVexMediaType = "application/vnd.cncf.openvex.v1+json";
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger _logger;
private readonly OciRegistryAuthorization _authorization;
private readonly OciOpenVexAttestationConnectorOptions _options;
public OciRegistryClient(
IHttpClientFactory httpClientFactory,
ILogger logger,
OciRegistryAuthorization authorization,
OciOpenVexAttestationConnectorOptions options)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_authorization = authorization ?? throw new ArgumentNullException(nameof(authorization));
_options = options ?? throw new ArgumentNullException(nameof(options));
}
public async Task<string?> ResolveDigestAsync(OciImageReference image, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(image);
if (image.HasDigest)
{
return image.Digest;
}
var requestUri = BuildRegistryUri(image, $"manifests/{EscapeReference(image.Tag ?? "latest")}");
async Task<HttpRequestMessage> RequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Head, requestUri);
request.Headers.Accept.ParseAdd(ManifestMediaType);
ApplyAuthentication(request);
return await Task.FromResult(request).ConfigureAwait(false);
}
using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("Failed to resolve digest for {Reference}; registry returned 404.", image.Canonical);
return null;
}
response.EnsureSuccessStatusCode();
}
if (response.Headers.TryGetValues("Docker-Content-Digest", out var values))
{
var digest = values.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(digest))
{
return digest;
}
}
// Manifest may have been returned without digest header; fall back to GET.
async Task<HttpRequestMessage> ManifestRequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Accept.ParseAdd(ManifestMediaType);
ApplyAuthentication(request);
return await Task.FromResult(request).ConfigureAwait(false);
}
using var manifestResponse = await SendAsync(ManifestRequestFactory, cancellationToken).ConfigureAwait(false);
manifestResponse.EnsureSuccessStatusCode();
if (manifestResponse.Headers.TryGetValues("Docker-Content-Digest", out var manifestValues))
{
return manifestValues.FirstOrDefault();
}
_logger.LogWarning("Registry {Registry} did not provide Docker-Content-Digest header for {Reference}.", image.Registry, image.Canonical);
return null;
}
public async Task<IReadOnlyList<OciArtifactDescriptor>> ListReferrersAsync(
OciImageReference image,
string subjectDigest,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(image);
ArgumentNullException.ThrowIfNull(subjectDigest);
var query = $"artifactType={Uri.EscapeDataString(ReferrersArtifactType)}";
var requestUri = BuildRegistryUri(image, $"referrers/{subjectDigest}", query);
async Task<HttpRequestMessage> RequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
ApplyAuthentication(request);
request.Headers.Accept.ParseAdd("application/json");
return await Task.FromResult(request).ConfigureAwait(false);
}
using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogDebug("Registry returned 404 for referrers on {Subject}.", subjectDigest);
return Array.Empty<OciArtifactDescriptor>();
}
response.EnsureSuccessStatusCode();
}
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var index = await JsonSerializer.DeserializeAsync<OciReferrerIndex>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
return index?.Referrers ?? Array.Empty<OciArtifactDescriptor>();
}
public async Task<OciAttestationDocument?> DownloadAttestationAsync(
OciImageReference image,
OciArtifactDescriptor descriptor,
string subjectDigest,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(image);
ArgumentNullException.ThrowIfNull(descriptor);
if (!IsSupportedDescriptor(descriptor))
{
return null;
}
var requestUri = BuildRegistryUri(image, $"blobs/{descriptor.Digest}");
async Task<HttpRequestMessage> RequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
ApplyAuthentication(request);
request.Headers.Accept.ParseAdd(descriptor.MediaType ?? "application/octet-stream");
return await Task.FromResult(request).ConfigureAwait(false);
}
using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("Registry returned 404 while downloading attestation {Digest} for {Subject}.", descriptor.Digest, subjectDigest);
return null;
}
response.EnsureSuccessStatusCode();
}
var buffer = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(image, descriptor, "registry", requestUri.ToString(), subjectDigest);
return new OciAttestationDocument(
requestUri,
buffer,
metadata,
subjectDigest,
descriptor.Digest,
descriptor.ArtifactType,
"registry");
}
private static bool IsSupportedDescriptor(OciArtifactDescriptor descriptor)
{
if (descriptor is null)
{
return false;
}
if (!string.IsNullOrWhiteSpace(descriptor.ArtifactType) &&
descriptor.ArtifactType.Equals(OpenVexMediaType, StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (!string.IsNullOrWhiteSpace(descriptor.MediaType) &&
(descriptor.MediaType.Equals(DsseMediaType, StringComparison.OrdinalIgnoreCase) ||
descriptor.MediaType.Equals(OpenVexMediaType, StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return false;
}
private async Task<HttpResponseMessage> SendAsync(
Func<Task<HttpRequestMessage>> requestFactory,
CancellationToken cancellationToken)
{
const int maxAttempts = 3;
TimeSpan delay = TimeSpan.FromSeconds(1);
Exception? lastError = null;
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
cancellationToken.ThrowIfCancellationRequested();
using var request = await requestFactory().ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(OciOpenVexAttestationConnectorOptions.HttpClientName);
try
{
var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return response;
}
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
if (_authorization.Mode == OciRegistryAuthMode.Anonymous && !_authorization.AllowAnonymousFallback)
{
var message = $"Registry request to {request.RequestUri} was unauthorized and anonymous fallback is disabled.";
response.Dispose();
throw new HttpRequestException(message);
}
lastError = new HttpRequestException($"Registry returned 401 Unauthorized for {request.RequestUri}.");
}
else if ((int)response.StatusCode >= 500 || response.StatusCode == (HttpStatusCode)429)
{
lastError = new HttpRequestException($"Registry returned status {(int)response.StatusCode} ({response.ReasonPhrase}) for {request.RequestUri}.");
}
else
{
response.EnsureSuccessStatusCode();
}
response.Dispose();
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
lastError = ex;
}
if (attempt < maxAttempts)
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 10));
}
}
throw new HttpRequestException("Failed to execute OCI registry request after multiple attempts.", lastError);
}
private void ApplyAuthentication(HttpRequestMessage request)
{
switch (_authorization.Mode)
{
case OciRegistryAuthMode.Basic when
!string.IsNullOrEmpty(_authorization.Username) &&
!string.IsNullOrEmpty(_authorization.Password):
var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_authorization.Username}:{_authorization.Password}"));
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
break;
case OciRegistryAuthMode.IdentityToken when !string.IsNullOrWhiteSpace(_authorization.IdentityToken):
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authorization.IdentityToken);
break;
case OciRegistryAuthMode.RefreshToken when !string.IsNullOrWhiteSpace(_authorization.RefreshToken):
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authorization.RefreshToken);
break;
default:
if (_authorization.Mode != OciRegistryAuthMode.Anonymous && !_authorization.AllowAnonymousFallback)
{
_logger.LogDebug("No authentication header applied for request to {Uri} (mode {Mode}).", request.RequestUri, _authorization.Mode);
}
break;
}
}
private Uri BuildRegistryUri(OciImageReference image, string relativePath, string? query = null)
{
var scheme = image.Scheme;
if (!string.Equals(scheme, "https", StringComparison.OrdinalIgnoreCase) && !_options.AllowHttpRegistries)
{
throw new InvalidOperationException($"HTTP access to registry '{image.Registry}' is disabled. Set AllowHttpRegistries to true to enable.");
}
var builder = new UriBuilder($"{scheme}://{image.Registry}")
{
Path = $"v2/{BuildRepositoryPath(image.Repository)}/{relativePath}"
};
if (!string.IsNullOrWhiteSpace(query))
{
builder.Query = query;
}
return builder.Uri;
}
private static string BuildRepositoryPath(string repository)
{
var segments = repository.Split('/', StringSplitOptions.RemoveEmptyEntries);
return string.Join('/', segments.Select(Uri.EscapeDataString));
}
private static string EscapeReference(string reference)
{
return Uri.EscapeDataString(reference);
}
private static ImmutableDictionary<string, string> BuildMetadata(
OciImageReference image,
OciArtifactDescriptor descriptor,
string sourceKind,
string sourcePath,
string subjectDigest)
{
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
builder["oci.image.registry"] = image.Registry;
builder["oci.image.repository"] = image.Repository;
builder["oci.image.reference"] = image.Canonical;
builder["oci.image.subjectDigest"] = subjectDigest;
builder["oci.attestation.sourceKind"] = sourceKind;
builder["oci.attestation.source"] = sourcePath;
builder["oci.attestation.artifactDigest"] = descriptor.Digest;
builder["oci.attestation.mediaType"] = descriptor.MediaType ?? string.Empty;
builder["oci.attestation.artifactType"] = descriptor.ArtifactType ?? string.Empty;
builder["oci.attestation.size"] = descriptor.Size.ToString(CultureInfo.InvariantCulture);
if (descriptor.Annotations is not null)
{
foreach (var annotation in descriptor.Annotations)
{
builder[$"oci.attestation.annotations.{annotation.Key}"] = annotation.Value;
}
}
return builder.ToImmutable();
}
}

View File

@@ -0,0 +1,221 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest;
public sealed class OciOpenVexAttestationConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor StaticDescriptor = new(
id: "excititor:oci.openvex.attest",
kind: VexProviderKind.Attestation,
displayName: "OCI OpenVEX Attestations")
{
Tags = ImmutableArray.Create("oci", "openvex", "attestation", "cosign", "offline"),
};
private readonly OciAttestationDiscoveryService _discoveryService;
private readonly OciAttestationFetcher _fetcher;
private readonly IEnumerable<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>> _validators;
private OciOpenVexAttestationConnectorOptions? _options;
private OciAttestationDiscoveryResult? _discovery;
public OciOpenVexAttestationConnector(
OciAttestationDiscoveryService discoveryService,
OciAttestationFetcher fetcher,
ILogger<OciOpenVexAttestationConnector> logger,
TimeProvider timeProvider,
IEnumerable<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>>? validators = null)
: base(StaticDescriptor, logger, timeProvider)
{
_discoveryService = discoveryService ?? throw new ArgumentNullException(nameof(discoveryService));
_fetcher = fetcher ?? throw new ArgumentNullException(nameof(fetcher));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_discovery = await _discoveryService.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Resolved OCI attestation targets.", new Dictionary<string, object?>
{
["targets"] = _discovery.Targets.Length,
["offlinePreferred"] = _discovery.PreferOffline,
["allowNetworkFallback"] = _discovery.AllowNetworkFallback,
["authMode"] = _discovery.RegistryAuthorization.Mode.ToString(),
["cosignMode"] = _discovery.CosignAuthority.Mode.ToString(),
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_discovery is null)
{
_discovery = await _discoveryService.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
}
var documentCount = 0;
await foreach (var document in _fetcher.FetchAsync(_discovery, _options, cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var verificationDocument = CreateRawDocument(
VexDocumentFormat.OciAttestation,
document.SourceUri,
document.Content,
document.Metadata);
var signatureMetadata = await context.SignatureVerifier.VerifyAsync(verificationDocument, cancellationToken).ConfigureAwait(false);
if (signatureMetadata is not null)
{
LogConnectorEvent(LogLevel.Debug, "signature", "Signature metadata captured for attestation.", new Dictionary<string, object?>
{
["subject"] = signatureMetadata.Subject,
["type"] = signatureMetadata.Type,
});
}
var enrichedMetadata = BuildProvenanceMetadata(document, signatureMetadata);
var rawDocument = CreateRawDocument(
VexDocumentFormat.OciAttestation,
document.SourceUri,
document.Content,
enrichedMetadata);
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
documentCount++;
yield return rawDocument;
}
LogConnectorEvent(LogLevel.Information, "fetch", "OCI attestation fetch completed.", new Dictionary<string, object?>
{
["documents"] = documentCount,
["since"] = context.Since?.ToString("O"),
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("Attestation documents rely on dedicated normalizers, to be wired in EXCITITOR-CONN-OCI-01-002.");
public OciAttestationDiscoveryResult? GetCachedDiscovery() => _discovery;
private ImmutableDictionary<string, string> BuildProvenanceMetadata(OciAttestationDocument document, VexSignatureMetadata? signature)
{
var builder = document.Metadata.ToBuilder();
if (!string.IsNullOrWhiteSpace(document.SourceKind))
{
builder["vex.provenance.sourceKind"] = document.SourceKind;
}
if (!string.IsNullOrWhiteSpace(document.SubjectDigest))
{
builder["vex.provenance.subjectDigest"] = document.SubjectDigest!;
}
if (!string.IsNullOrWhiteSpace(document.ArtifactDigest))
{
builder["vex.provenance.artifactDigest"] = document.ArtifactDigest!;
}
if (!string.IsNullOrWhiteSpace(document.ArtifactType))
{
builder["vex.provenance.artifactType"] = document.ArtifactType!;
}
if (_discovery is not null)
{
builder["vex.provenance.registryAuthMode"] = _discovery.RegistryAuthorization.Mode.ToString();
var registryAuthority = _discovery.RegistryAuthorization.RegistryAuthority;
if (string.IsNullOrWhiteSpace(registryAuthority))
{
if (builder.TryGetValue("oci.image.registry", out var metadataRegistry) && !string.IsNullOrWhiteSpace(metadataRegistry))
{
registryAuthority = metadataRegistry;
}
}
if (!string.IsNullOrWhiteSpace(registryAuthority))
{
builder["vex.provenance.registryAuthority"] = registryAuthority!;
}
builder["vex.provenance.cosign.mode"] = _discovery.CosignAuthority.Mode.ToString();
if (_discovery.CosignAuthority.Keyless is not null)
{
var keyless = _discovery.CosignAuthority.Keyless;
builder["vex.provenance.cosign.issuer"] = keyless!.Issuer;
builder["vex.provenance.cosign.subject"] = keyless.Subject;
if (keyless.FulcioUrl is not null)
{
builder["vex.provenance.cosign.fulcioUrl"] = keyless.FulcioUrl!.ToString();
}
if (keyless.RekorUrl is not null)
{
builder["vex.provenance.cosign.rekorUrl"] = keyless.RekorUrl!.ToString();
}
}
else if (_discovery.CosignAuthority.KeyPair is not null)
{
var keyPair = _discovery.CosignAuthority.KeyPair;
builder["vex.provenance.cosign.keyPair"] = "true";
if (keyPair!.RekorUrl is not null)
{
builder["vex.provenance.cosign.rekorUrl"] = keyPair.RekorUrl!.ToString();
}
}
}
if (signature is not null)
{
builder["vex.signature.type"] = signature.Type;
if (!string.IsNullOrWhiteSpace(signature.Subject))
{
builder["vex.signature.subject"] = signature.Subject!;
}
if (!string.IsNullOrWhiteSpace(signature.Issuer))
{
builder["vex.signature.issuer"] = signature.Issuer!;
}
if (!string.IsNullOrWhiteSpace(signature.KeyId))
{
builder["vex.signature.keyId"] = signature.KeyId!;
}
if (signature.VerifiedAt is not null)
{
builder["vex.signature.verifiedAt"] = signature.VerifiedAt.Value.ToString("O");
}
if (!string.IsNullOrWhiteSpace(signature.TransparencyLogReference))
{
builder["vex.signature.transparencyLogReference"] = signature.TransparencyLogReference!;
}
}
return builder.ToImmutable();
}
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-OCI-01-001 OCI discovery & auth plumbing|Team Excititor Connectors OCI|EXCITITOR-CONN-ABS-01-001|DONE (2025-10-18) Added connector skeleton, options/validators, discovery caching, cosign/auth descriptors, offline bundle resolution, DI wiring, and regression tests.|
|EXCITITOR-CONN-OCI-01-002 Attestation fetch & verify loop|Team Excititor Connectors OCI|EXCITITOR-CONN-OCI-01-001, EXCITITOR-ATTEST-01-002|DONE (2025-10-18) Added offline/registry fetch services, DSSE retrieval with retries, signature verification callout, and raw persistence coverage.|
|EXCITITOR-CONN-OCI-01-003 Provenance metadata & policy hooks|Team Excititor Connectors OCI|EXCITITOR-CONN-OCI-01-002, EXCITITOR-POLICY-01-001|DONE (2025-10-18) Enriched attestation metadata with provenance hints, cosign expectations, registry auth context, and signature diagnostics for policy consumption.|

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector for Oracle CSAF advisories, including CPU and other bulletin releases, projecting documents into raw storage for normalization.
## Scope
- Discovery of Oracle CSAF catalogue, navigation of quarterly CPU bundles, and delta detection.
- HTTP fetch with retry/backoff, checksum validation, and deduplication across revisions.
- Mapping Oracle advisory metadata (CPU ID, component families) into connector context.
- Publishing trust metadata (PGP keys/cosign options) aligned with policy expectations.
## Participants
- Worker orchestrates regular pulls respecting Oracle publication cadence; WebService offers manual triggers.
- CSAF normalizer processes raw documents to claims.
- Policy engine leverages trust metadata and provenance hints.
## Interfaces & contracts
- Implements `IVexConnector` using shared abstractions for HTTP/resume and telemetry.
- Configuration options for CPU schedule, credentials (if required), and offline snapshot ingestion.
## In/Out of scope
In: fetching, metadata mapping, raw persistence, trust hints.
Out: normalization, storage internals, export/attestation flows.
## Observability & security expectations
- Log CPU release windows, document counts, and fetch durations; redact any secrets.
- Emit metrics for deduped vs new documents and quarantine rates.
## Tests
- Harness tests with mocked Oracle catalogues will live in `../StellaOps.Excititor.Connectors.Oracle.CSAF.Tests`.

View File

@@ -0,0 +1,85 @@
using System;
using System.IO;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
public sealed class OracleConnectorOptions
{
public const string HttpClientName = "excititor.connector.oracle.catalog";
/// <summary>
/// Oracle CSAF catalog endpoint hosting advisory metadata.
/// </summary>
public Uri CatalogUri { get; set; } = new("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json");
/// <summary>
/// Optional CPU calendar endpoint providing upcoming release dates.
/// </summary>
public Uri? CpuCalendarUri { get; set; }
/// <summary>
/// Duration the discovery metadata should be cached before refresh.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6);
/// <summary>
/// When true, the loader will prefer offline snapshot data over network fetches.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Optional file path for persisting or ingesting catalog snapshots.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// Enables writing fresh catalog responses to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
/// <summary>
/// Optional request delay when iterating catalogue entries (for rate limiting).
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public void Validate(IFileSystem? fileSystem = null)
{
if (CatalogUri is null || !CatalogUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CatalogUri must be an absolute URI.");
}
if (CatalogUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("CatalogUri must use HTTP or HTTPS.");
}
if (CpuCalendarUri is not null && (!CpuCalendarUri.IsAbsoluteUri || CpuCalendarUri.Scheme is not ("http" or "https")))
{
throw new InvalidOperationException("CpuCalendarUri must be an absolute HTTP(S) URI when provided.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("MetadataCacheDuration must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
public sealed class OracleConnectorOptionsValidator : IVexConnectorOptionsValidator<OracleConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public OracleConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(VexConnectorDescriptor descriptor, OracleConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.DependencyInjection;
public static class OracleConnectorServiceCollectionExtensions
{
public static IServiceCollection AddOracleCsafConnector(this IServiceCollection services, Action<OracleConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<OracleConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddSingleton<IVexConnectorOptionsValidator<OracleConnectorOptions>, OracleConnectorOptionsValidator>();
services.AddHttpClient(OracleConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Oracle.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<OracleCatalogLoader>();
services.AddSingleton<IVexConnector, OracleCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,418 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
public sealed class OracleCatalogLoader
{
public const string CachePrefix = "StellaOps.Excititor.Connectors.Oracle.CSAF.Catalog";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<OracleCatalogLoader> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _semaphore = new(1, 1);
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
public OracleCatalogLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IFileSystem fileSystem,
ILogger<OracleCatalogLoader> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<OracleCatalogResult> LoadAsync(OracleConnectorOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
options.Validate(_fileSystem);
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
CacheEntry? entry = null;
if (options.PreferOfflineSnapshot)
{
entry = LoadFromOffline(options);
if (entry is null)
{
throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline snapshot was found or could be loaded.");
}
}
else
{
entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false)
?? LoadFromOffline(options);
}
if (entry is null)
{
throw new InvalidOperationException("Unable to load Oracle CSAF catalog from network or offline snapshot.");
}
var expiration = entry.MetadataCacheDuration == TimeSpan.Zero
? (DateTimeOffset?)null
: _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration);
var cacheEntryOptions = new MemoryCacheEntryOptions();
if (expiration.HasValue)
{
cacheEntryOptions.AbsoluteExpiration = expiration.Value;
}
_memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheEntryOptions);
return entry.ToResult(fromCache: false);
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(OracleConnectorOptions options, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName);
using var response = await client.GetAsync(options.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var catalogPayload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
string? calendarPayload = null;
if (options.CpuCalendarUri is not null)
{
try
{
using var calendarResponse = await client.GetAsync(options.CpuCalendarUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
calendarResponse.EnsureSuccessStatusCode();
calendarPayload = await calendarResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to fetch Oracle CPU calendar from {Uri}; continuing without schedule.", options.CpuCalendarUri);
}
}
var metadata = ParseMetadata(catalogPayload, calendarPayload);
var fetchedAt = _timeProvider.GetUtcNow();
var entry = new CacheEntry(metadata, fetchedAt, fetchedAt, options.MetadataCacheDuration, false);
PersistSnapshotIfNeeded(options, metadata, fetchedAt);
return entry;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to fetch Oracle CSAF catalog from {Uri}; attempting offline fallback if available.", options.CatalogUri);
return null;
}
}
private CacheEntry? LoadFromOffline(OracleConnectorOptions options)
{
if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(options.OfflineSnapshotPath))
{
_logger.LogWarning("Oracle offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath);
var snapshot = JsonSerializer.Deserialize<OracleCatalogSnapshot>(payload, _serializerOptions);
if (snapshot is null)
{
throw new InvalidOperationException("Offline snapshot payload was empty.");
}
return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Oracle CSAF catalog from offline snapshot {Path}.", options.OfflineSnapshotPath);
return null;
}
}
private OracleCatalogMetadata ParseMetadata(string catalogPayload, string? calendarPayload)
{
if (string.IsNullOrWhiteSpace(catalogPayload))
{
throw new InvalidOperationException("Oracle catalog payload was empty.");
}
using var document = JsonDocument.Parse(catalogPayload);
var root = document.RootElement;
var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated)
? generated
: _timeProvider.GetUtcNow();
var entries = ParseEntries(root);
var schedule = ParseSchedule(root);
if (!string.IsNullOrWhiteSpace(calendarPayload))
{
schedule = MergeSchedule(schedule, calendarPayload);
}
return new OracleCatalogMetadata(generatedAt, entries, schedule);
}
private ImmutableArray<OracleCatalogEntry> ParseEntries(JsonElement root)
{
if (!root.TryGetProperty("catalog", out var catalogElement) || catalogElement.ValueKind is not JsonValueKind.Array)
{
return ImmutableArray<OracleCatalogEntry>.Empty;
}
var builder = ImmutableArray.CreateBuilder<OracleCatalogEntry>();
foreach (var entry in catalogElement.EnumerateArray())
{
var id = entry.TryGetProperty("id", out var idElement) && idElement.ValueKind == JsonValueKind.String ? idElement.GetString() : null;
var title = entry.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String ? titleElement.GetString() : null;
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(title))
{
continue;
}
DateTimeOffset publishedAt = default;
if (entry.TryGetProperty("published", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(publishedElement.GetString(), out var publishedParsed))
{
publishedAt = publishedParsed;
}
string? revision = null;
if (entry.TryGetProperty("revision", out var revisionElement) && revisionElement.ValueKind == JsonValueKind.String)
{
revision = revisionElement.GetString();
}
ImmutableArray<string> products = ImmutableArray<string>.Empty;
if (entry.TryGetProperty("products", out var productsElement))
{
products = ParseStringArray(productsElement);
}
Uri? documentUri = null;
string? sha256 = null;
long? size = null;
if (entry.TryGetProperty("document", out var documentElement) && documentElement.ValueKind == JsonValueKind.Object)
{
if (documentElement.TryGetProperty("url", out var urlElement) && urlElement.ValueKind == JsonValueKind.String && Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var parsedUri))
{
documentUri = parsedUri;
}
if (documentElement.TryGetProperty("sha256", out var hashElement) && hashElement.ValueKind == JsonValueKind.String)
{
sha256 = hashElement.GetString();
}
if (documentElement.TryGetProperty("size", out var sizeElement) && sizeElement.ValueKind == JsonValueKind.Number && sizeElement.TryGetInt64(out var parsedSize))
{
size = parsedSize;
}
}
if (documentUri is null)
{
continue;
}
builder.Add(new OracleCatalogEntry(id!, title!, documentUri, publishedAt, revision, sha256, size, products));
}
return builder.ToImmutable();
}
private ImmutableArray<OracleCpuRelease> ParseSchedule(JsonElement root)
{
if (!root.TryGetProperty("schedule", out var scheduleElement) || scheduleElement.ValueKind is not JsonValueKind.Array)
{
return ImmutableArray<OracleCpuRelease>.Empty;
}
var builder = ImmutableArray.CreateBuilder<OracleCpuRelease>();
foreach (var item in scheduleElement.EnumerateArray())
{
var window = item.TryGetProperty("window", out var windowElement) && windowElement.ValueKind == JsonValueKind.String ? windowElement.GetString() : null;
if (string.IsNullOrWhiteSpace(window))
{
continue;
}
DateTimeOffset releaseDate = default;
if (item.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed))
{
releaseDate = parsed;
}
builder.Add(new OracleCpuRelease(window!, releaseDate));
}
return builder.ToImmutable();
}
private ImmutableArray<OracleCpuRelease> MergeSchedule(ImmutableArray<OracleCpuRelease> existing, string calendarPayload)
{
try
{
using var document = JsonDocument.Parse(calendarPayload);
var root = document.RootElement;
if (!root.TryGetProperty("cpuWindows", out var windowsElement) || windowsElement.ValueKind is not JsonValueKind.Array)
{
return existing;
}
var builder = existing.ToBuilder();
var known = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var item in builder)
{
known.Add(item.Window);
}
foreach (var windowElement in windowsElement.EnumerateArray())
{
var name = windowElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
if (!known.Add(name))
{
continue;
}
DateTimeOffset releaseDate = default;
if (windowElement.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed))
{
releaseDate = parsed;
}
builder.Add(new OracleCpuRelease(name!, releaseDate));
}
return builder.ToImmutable();
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to parse Oracle CPU calendar payload; continuing with existing schedule data.");
return existing;
}
}
private ImmutableArray<string> ParseStringArray(JsonElement element)
{
if (element.ValueKind is not JsonValueKind.Array)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var item in element.EnumerateArray())
{
if (item.ValueKind == JsonValueKind.String)
{
var value = item.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value);
}
}
}
return builder.ToImmutable();
}
private void PersistSnapshotIfNeeded(OracleConnectorOptions options, OracleCatalogMetadata metadata, DateTimeOffset fetchedAt)
{
if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return;
}
try
{
var snapshot = new OracleCatalogSnapshot(metadata, fetchedAt);
var payload = JsonSerializer.Serialize(snapshot, _serializerOptions);
_fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath);
}
}
private static string CreateCacheKey(OracleConnectorOptions options)
=> $"{CachePrefix}:{options.CatalogUri}:{options.CpuCalendarUri}";
private sealed record CacheEntry(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot)
{
public bool IsExpired(DateTimeOffset now)
=> MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration;
public OracleCatalogResult ToResult(bool fromCache)
=> new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot);
}
private sealed record OracleCatalogSnapshot(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt);
}
public sealed record OracleCatalogMetadata(
DateTimeOffset GeneratedAt,
ImmutableArray<OracleCatalogEntry> Entries,
ImmutableArray<OracleCpuRelease> CpuSchedule);
public sealed record OracleCatalogEntry(
string Id,
string Title,
Uri DocumentUri,
DateTimeOffset PublishedAt,
string? Revision,
string? Sha256,
long? Size,
ImmutableArray<string> Products);
public sealed record OracleCpuRelease(string Window, DateTimeOffset ReleaseDate);
public sealed record OracleCatalogResult(
OracleCatalogMetadata Metadata,
DateTimeOffset FetchedAt,
bool FromCache,
bool FromOfflineSnapshot);

View File

@@ -0,0 +1,360 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF;
public sealed class OracleCsafConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:oracle",
kind: VexProviderKind.Vendor,
displayName: "Oracle CSAF")
{
Tags = ImmutableArray.Create("oracle", "csaf", "cpu"),
};
private readonly OracleCatalogLoader _catalogLoader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> _validators;
private OracleConnectorOptions? _options;
private OracleCatalogResult? _catalog;
public OracleCsafConnector(
OracleCatalogLoader catalogLoader,
IHttpClientFactory httpClientFactory,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> validators,
ILogger<OracleCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<OracleConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary<string, object?>
{
["catalogEntryCount"] = _catalog.Metadata.Entries.Length,
["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length,
["fromOffline"] = _catalog.FromOfflineSnapshot,
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
_catalog ??= await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
var entries = _catalog.Metadata.Entries
.OrderBy(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt)
.ToImmutableArray();
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var since = ResolveSince(context.Since, state?.LastUpdated);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var latestPublished = state?.LastUpdated ?? since ?? DateTimeOffset.MinValue;
var stateChanged = false;
var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName);
LogConnectorEvent(LogLevel.Information, "fetch.begin", "Starting Oracle CSAF catalogue iteration.", new Dictionary<string, object?>
{
["since"] = since?.ToString("O"),
["entryCount"] = entries.Length,
});
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
if (ShouldSkipEntry(entry, since))
{
continue;
}
var expectedDigest = NormalizeDigest(entry.Sha256);
if (expectedDigest is not null && digestSet.Contains(expectedDigest))
{
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
LogConnectorEvent(LogLevel.Debug, "fetch.skip.cached", "Skipping Oracle CSAF entry because digest already processed.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["digest"] = expectedDigest,
});
continue;
}
var rawDocument = await DownloadEntryAsync(client, entry, cancellationToken).ConfigureAwait(false);
if (rawDocument is null)
{
continue;
}
if (expectedDigest is not null && !string.Equals(rawDocument.Digest, expectedDigest, StringComparison.OrdinalIgnoreCase))
{
LogConnectorEvent(LogLevel.Warning, "fetch.checksum_mismatch", "Oracle CSAF document checksum mismatch; document skipped.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["expected"] = expectedDigest,
["actual"] = rawDocument.Digest,
["documentUri"] = entry.DocumentUri.ToString(),
});
continue;
}
if (!digestSet.Add(rawDocument.Digest))
{
LogConnectorEvent(LogLevel.Debug, "fetch.skip.duplicate", "Oracle CSAF document digest already ingested.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["digest"] = rawDocument.Digest,
});
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
LogConnectorEvent(LogLevel.Information, "fetch.document_ingested", "Oracle CSAF document stored.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["digest"] = rawDocument.Digest,
["documentUri"] = entry.DocumentUri.ToString(),
["publishedAt"] = entry.PublishedAt.ToString("O"),
});
yield return rawDocument;
if (_options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
if (stateChanged)
{
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latestPublished == DateTimeOffset.MinValue ? baseState.LastUpdated : latestPublished,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
var ingestedCount = digestList.Count - knownDigests.Length;
LogConnectorEvent(LogLevel.Information, "fetch.complete", "Oracle CSAF fetch completed.", new Dictionary<string, object?>
{
["stateChanged"] = stateChanged,
["documentsProcessed"] = ingestedCount,
["latestPublished"] = latestPublished == DateTimeOffset.MinValue ? null : latestPublished.ToString("O"),
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers.");
public OracleCatalogResult? GetCachedCatalog() => _catalog;
private static DateTimeOffset? ResolveSince(DateTimeOffset? contextSince, DateTimeOffset? stateSince)
{
if (contextSince is null)
{
return stateSince;
}
if (stateSince is null)
{
return contextSince;
}
return stateSince > contextSince ? stateSince : contextSince;
}
private static bool ShouldSkipEntry(OracleCatalogEntry entry, DateTimeOffset? since)
{
if (since is null)
{
return false;
}
if (entry.PublishedAt == default)
{
return false;
}
return entry.PublishedAt <= since;
}
private async Task<VexRawDocument?> DownloadEntryAsync(HttpClient client, OracleCatalogEntry entry, CancellationToken cancellationToken)
{
if (entry.DocumentUri is null)
{
LogConnectorEvent(LogLevel.Warning, "fetch.skip.missing_uri", "Oracle CSAF entry missing document URI; skipping.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
});
return null;
}
var payload = await DownloadWithRetryAsync(client, entry.DocumentUri, cancellationToken).ConfigureAwait(false);
if (payload is null)
{
return null;
}
var metadata = BuildMetadata(builder =>
{
builder.Add("oracle.csaf.entryId", entry.Id);
builder.Add("oracle.csaf.title", entry.Title);
builder.Add("oracle.csaf.revision", entry.Revision);
if (entry.PublishedAt != default)
{
builder.Add("oracle.csaf.published", entry.PublishedAt.ToString("O"));
}
builder.Add("oracle.csaf.sha256", NormalizeDigest(entry.Sha256));
builder.Add("oracle.csaf.size", entry.Size?.ToString(CultureInfo.InvariantCulture));
if (!entry.Products.IsDefaultOrEmpty)
{
builder.Add("oracle.csaf.products", string.Join(",", entry.Products));
}
});
return CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload.AsMemory(), metadata);
}
private async Task<byte[]?> DownloadWithRetryAsync(HttpClient client, Uri uri, CancellationToken cancellationToken)
{
const int maxAttempts = 3;
var delay = TimeSpan.FromSeconds(1);
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using var response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (IsTransient(response.StatusCode) && attempt < maxAttempts)
{
LogConnectorEvent(LogLevel.Warning, "fetch.retry.status", "Oracle CSAF document request returned transient status; retrying.", new Dictionary<string, object?>
{
["status"] = (int)response.StatusCode,
["attempt"] = attempt,
["uri"] = uri.ToString(),
});
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = delay + delay;
continue;
}
response.EnsureSuccessStatusCode();
}
var bytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
return bytes;
}
catch (Exception ex) when (IsTransient(ex) && attempt < maxAttempts)
{
LogConnectorEvent(LogLevel.Warning, "fetch.retry.exception", "Oracle CSAF document request failed; retrying.", new Dictionary<string, object?>
{
["attempt"] = attempt,
["uri"] = uri.ToString(),
["exception"] = ex.GetType().Name,
});
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = delay + delay;
}
}
LogConnectorEvent(LogLevel.Error, "fetch.failed", "Oracle CSAF document could not be retrieved after retries.", new Dictionary<string, object?>
{
["uri"] = uri.ToString(),
});
return null;
}
private static bool IsTransient(Exception exception)
=> exception is HttpRequestException or IOException or TaskCanceledException;
private static bool IsTransient(HttpStatusCode statusCode)
{
var status = (int)statusCode;
return status is >= 500 or 408 or 429;
}
private static string? NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return null;
}
var trimmed = digest.Trim();
if (!trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
trimmed = "sha256:" + trimmed;
}
return trimmed.ToLowerInvariant();
}
private static DateTimeOffset UpdateLatest(DateTimeOffset current, DateTimeOffset published)
{
if (published == default)
{
return current;
}
return published > current ? published : current;
}
}

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-ORACLE-01-001 Oracle CSAF catalogue discovery|Team Excititor Connectors Oracle|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-19)** Implemented cached Oracle CSAF catalog loader with CPU calendar merge, offline snapshot ingest/persist, options validation + DI wiring, and regression tests; prerequisite EXCITITOR-CONN-ABS-01-001 verified DONE per Sprint 5 log (2025-10-19).|
|EXCITITOR-CONN-ORACLE-01-002 CSAF download & dedupe pipeline|Team Excititor Connectors Oracle|EXCITITOR-CONN-ORACLE-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-19)** Added Oracle CSAF fetch loop with retry/backoff, checksum validation, resume-aware state persistence, digest dedupe, configurable throttling, and raw storage wiring; regression tests cover new ingestion and mismatch handling.|
|EXCITITOR-CONN-ORACLE-01-003 Trust metadata + provenance|Team Excititor Connectors Oracle|EXCITITOR-CONN-ORACLE-01-002, EXCITITOR-POLICY-01-001|TODO Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting.|

View File

@@ -0,0 +1,25 @@
# AGENTS
## Role
Connector for Red Hat CSAF VEX feeds, fetching provider metadata, CSAF documents, and projecting them into raw storage for normalization.
## Scope
- Discovery via `/.well-known/csaf/provider-metadata.json`, scheduling windows, and ETag-aware HTTP fetches.
- `RedHatProviderMetadataLoader` handles `.well-known` metadata with caching, schema validation, and offline snapshots.
- `RedHatCsafConnector` consumes ROLIE feeds to fetch incremental CSAF documents, honours `context.Since`, and streams raw advisories to storage.
- Mapping Red Hat CSAF specifics (product tree aliases, RHSA identifiers, revision history) into raw documents.
- Emitting structured telemetry and resume markers for incremental pulls.
- Supplying Red Hat-specific trust overrides and provenance hints to normalization.
## Participants
- Worker schedules pulls using this connector; WebService triggers ad-hoc runs.
- CSAF normalizer consumes fetched documents to produce claims.
- Policy/consensus rely on Red Hat trust metadata captured here.
## Interfaces & contracts
- Implements `IVexConnector` with Red Hat-specific options (parallelism, token auth if configured).
- Uses abstractions from `StellaOps.Excititor.Connectors.Abstractions` for HTTP/resume helpers.
## In/Out of scope
In: data acquisition, HTTP retries, raw document persistence, provider metadata population.
Out: normalization, storage internals, attestation, general connector abstractions (covered elsewhere).
## Observability & security expectations
- Log provider metadata URL, revision ids, fetch durations; redact tokens.
- Emit counters for documents fetched, skipped (304), quarantined.
## Tests
- Connector harness tests (mock HTTP) and resume regression cases will live in `../StellaOps.Excititor.Connectors.RedHat.CSAF.Tests`.

View File

@@ -0,0 +1,104 @@
using System.Collections.Generic;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
public sealed class RedHatConnectorOptions
{
public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json");
/// <summary>
/// HTTP client name registered for the connector.
/// </summary>
public const string HttpClientName = "excititor.connector.redhat";
/// <summary>
/// URI of the CSAF provider metadata document.
/// </summary>
public Uri MetadataUri { get; set; } = DefaultMetadataUri;
/// <summary>
/// Duration to cache loaded metadata before refreshing.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(1);
/// <summary>
/// Optional file path used to store or source offline metadata snapshots.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// When true, the loader prefers the offline snapshot without attempting a network fetch.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Enables writing fresh metadata responses to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
/// <summary>
/// Explicit trust weight override applied to the provider entry.
/// </summary>
public double TrustWeight { get; set; } = 1.0;
/// <summary>
/// Sigstore/Cosign issuer used to verify CSAF signatures, if published.
/// </summary>
public string? CosignIssuer { get; set; } = "https://access.redhat.com";
/// <summary>
/// Identity pattern matched against the Cosign certificate subject.
/// </summary>
public string? CosignIdentityPattern { get; set; } = "^https://access\\.redhat\\.com/.+$";
/// <summary>
/// Optional list of PGP fingerprints recognised for Red Hat CSAF artifacts.
/// </summary>
public IList<string> PgpFingerprints { get; } = new List<string>();
public void Validate(IFileSystem? fileSystem = null)
{
if (MetadataUri is null || !MetadataUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Metadata URI must be absolute.");
}
if (MetadataUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("Metadata URI must use HTTP or HTTPS.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("Metadata cache duration must be positive.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight) || TrustWeight <= 0)
{
TrustWeight = 1.0;
}
else if (TrustWeight > 1.0)
{
TrustWeight = 1.0;
}
if (CosignIssuer is not null)
{
if (string.IsNullOrWhiteSpace(CosignIdentityPattern))
{
throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified.");
}
}
}
}

View File

@@ -0,0 +1,45 @@
using System.Net;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
public static class RedHatConnectorServiceCollectionExtensions
{
public static IServiceCollection AddRedHatCsafConnector(this IServiceCollection services, Action<RedHatConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.AddOptions<RedHatConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
})
.PostConfigure(options => options.Validate());
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddHttpClient(RedHatConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.RedHat/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<RedHatProviderMetadataLoader>();
services.AddSingleton<IVexConnector, RedHatCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,312 @@
using System.Collections.Immutable;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
public sealed class RedHatProviderMetadataLoader
{
public const string CacheKey = "StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
private readonly ILogger<RedHatProviderMetadataLoader> _logger;
private readonly RedHatConnectorOptions _options;
private readonly IFileSystem _fileSystem;
private readonly JsonSerializerOptions _serializerOptions;
private readonly SemaphoreSlim _refreshSemaphore = new(1, 1);
public RedHatProviderMetadataLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IOptions<RedHatConnectorOptions> options,
ILogger<RedHatProviderMetadataLoader> logger,
IFileSystem? fileSystem = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_cache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_fileSystem = fileSystem ?? new FileSystem();
_serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
}
public async Task<RedHatProviderMetadataResult> LoadAsync(CancellationToken cancellationToken)
{
if (_cache.TryGetValue<CacheEntry>(CacheKey, out var cached) && cached is { } cachedEntry && !cachedEntry.IsExpired())
{
_logger.LogDebug("Returning cached Red Hat provider metadata (expires {Expires}).", cachedEntry.ExpiresAt);
return new RedHatProviderMetadataResult(cachedEntry.Provider, cachedEntry.FetchedAt, true, cachedEntry.FromOffline);
}
await _refreshSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_cache.TryGetValue<CacheEntry>(CacheKey, out cached) && cached is { } cachedAfterLock && !cachedAfterLock.IsExpired())
{
return new RedHatProviderMetadataResult(cachedAfterLock.Provider, cachedAfterLock.FetchedAt, true, cachedAfterLock.FromOffline);
}
CacheEntry? previous = cached;
// Attempt live fetch unless offline preferred.
if (!_options.PreferOfflineSnapshot)
{
var httpResult = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false);
if (httpResult is not null)
{
StoreCache(httpResult);
return new RedHatProviderMetadataResult(httpResult.Provider, httpResult.FetchedAt, false, false);
}
}
var offlineResult = TryLoadFromOffline();
if (offlineResult is not null)
{
var offlineEntry = offlineResult with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
FromOffline = true,
};
StoreCache(offlineEntry);
return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true);
}
throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot.");
}
finally
{
_refreshSemaphore.Release();
}
}
private void StoreCache(CacheEntry entry)
{
var cacheEntryOptions = new MemoryCacheEntryOptions
{
AbsoluteExpiration = entry.ExpiresAt,
};
_cache.Set(CacheKey, entry, cacheEntryOptions);
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri);
if (!string.IsNullOrWhiteSpace(previous?.ETag))
{
if (EntityTagHeaderValue.TryParse(previous.ETag, out var etag))
{
request.Headers.IfNoneMatch.Add(etag);
}
}
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotModified && previous is not null)
{
_logger.LogDebug("Red Hat provider metadata not modified (etag {ETag}).", previous.ETag);
return previous with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
};
}
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var provider = ParseAndValidate(payload);
var etagHeader = response.Headers.ETag?.ToString();
if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
try
{
_fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
}
return new CacheEntry(
provider,
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
etagHeader,
FromOffline: false);
}
catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot)
{
_logger.LogWarning(ex, "Failed to fetch Red Hat provider metadata from {Uri}, will attempt offline snapshot.", _options.MetadataUri);
return null;
}
}
private CacheEntry? TryLoadFromOffline()
{
if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath))
{
_logger.LogWarning("Offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath);
var provider = ParseAndValidate(payload);
return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, ETag: null, FromOffline: true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Red Hat provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath);
return null;
}
}
private VexProvider ParseAndValidate(string payload)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Provider metadata payload was empty.");
}
ProviderMetadataDocument? document;
try
{
document = JsonSerializer.Deserialize<ProviderMetadataDocument>(payload, _serializerOptions);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Provider metadata payload could not be parsed.", ex);
}
if (document is null)
{
throw new InvalidOperationException("Provider metadata payload was null after parsing.");
}
if (document.Metadata?.Provider?.Name is null)
{
throw new InvalidOperationException("Provider metadata missing provider name.");
}
var distributions = document.Distributions?
.Select(static d => d.Directory)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Select(static s => CreateUri(s!, nameof(ProviderMetadataDistribution.Directory)))
.ToImmutableArray() ?? ImmutableArray<Uri>.Empty;
if (distributions.IsDefaultOrEmpty)
{
throw new InvalidOperationException("Provider metadata did not include any valid distribution directories.");
}
Uri? rolieFeed = null;
if (document.Rolie?.Feeds is not null)
{
foreach (var feed in document.Rolie.Feeds)
{
if (!string.IsNullOrWhiteSpace(feed.Url))
{
rolieFeed = CreateUri(feed.Url, "rolie.feeds[].url");
break;
}
}
}
var trust = BuildTrust();
return new VexProvider(
id: "excititor:redhat",
displayName: document.Metadata.Provider.Name,
kind: VexProviderKind.Distro,
baseUris: distributions,
discovery: new VexProviderDiscovery(_options.MetadataUri, rolieFeed),
trust: trust);
}
private VexProviderTrust BuildTrust()
{
VexCosignTrust? cosign = null;
if (!string.IsNullOrWhiteSpace(_options.CosignIssuer) && !string.IsNullOrWhiteSpace(_options.CosignIdentityPattern))
{
cosign = new VexCosignTrust(_options.CosignIssuer!, _options.CosignIdentityPattern!);
}
return new VexProviderTrust(
_options.TrustWeight,
cosign,
_options.PgpFingerprints);
}
private static Uri CreateUri(string value, string propertyName)
{
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException($"Provider metadata field '{propertyName}' must be an absolute HTTP(S) URI.");
}
return uri;
}
private sealed record ProviderMetadataDocument(
[property: JsonPropertyName("metadata")] ProviderMetadata? Metadata,
[property: JsonPropertyName("distributions")] IReadOnlyList<ProviderMetadataDistribution>? Distributions,
[property: JsonPropertyName("rolie")] ProviderMetadataRolie? Rolie);
private sealed record ProviderMetadata(
[property: JsonPropertyName("provider")] ProviderMetadataProvider? Provider);
private sealed record ProviderMetadataProvider(
[property: JsonPropertyName("name")] string? Name);
private sealed record ProviderMetadataDistribution(
[property: JsonPropertyName("directory")] string? Directory);
private sealed record ProviderMetadataRolie(
[property: JsonPropertyName("feeds")] IReadOnlyList<ProviderMetadataRolieFeed>? Feeds);
private sealed record ProviderMetadataRolieFeed(
[property: JsonPropertyName("url")] string? Url);
private sealed record CacheEntry(
VexProvider Provider,
DateTimeOffset FetchedAt,
DateTimeOffset ExpiresAt,
string? ETag,
bool FromOffline)
{
public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt;
}
}
public sealed record RedHatProviderMetadataResult(
VexProvider Provider,
DateTimeOffset FetchedAt,
bool FromCache,
bool FromOfflineSnapshot);

View File

@@ -0,0 +1,196 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF;
public sealed class RedHatCsafConnector : VexConnectorBase
{
private readonly RedHatProviderMetadataLoader _metadataLoader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IVexConnectorStateRepository _stateRepository;
public RedHatCsafConnector(
VexConnectorDescriptor descriptor,
RedHatProviderMetadataLoader metadataLoader,
IHttpClientFactory httpClientFactory,
IVexConnectorStateRepository stateRepository,
ILogger<RedHatCsafConnector> logger,
TimeProvider timeProvider)
: base(descriptor, logger, timeProvider)
{
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
}
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
// No connector-specific settings yet.
return ValueTask.CompletedTask;
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
if (metadataResult.Provider.Discovery.RolIeService is null)
{
throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed.");
}
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var sinceTimestamp = context.Since;
if (state?.LastUpdated is { } persisted && (sinceTimestamp is null || persisted > sinceTimestamp))
{
sinceTimestamp = persisted;
}
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestList = new List<string>(knownDigests);
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue;
var stateChanged = false;
foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false))
{
if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp)
{
continue;
}
if (entry.DocumentUri is null)
{
Logger.LogDebug("Skipping ROLIE entry {Id} because no document link was provided.", entry.Id);
continue;
}
var rawDocument = await DownloadCsafDocumentAsync(entry, cancellationToken).ConfigureAwait(false);
if (!digestSet.Add(rawDocument.Digest))
{
Logger.LogDebug("Skipping CSAF document {Uri} because digest {Digest} was already processed.", rawDocument.SourceUri, rawDocument.Digest);
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
if (entry.Updated is DateTimeOffset entryUpdated && entryUpdated > latestUpdated)
{
latestUpdated = entryUpdated;
}
yield return rawDocument;
}
if (stateChanged)
{
var newLastUpdated = latestUpdated == DateTimeOffset.MinValue ? state?.LastUpdated : latestUpdated;
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var updatedState = baseState with
{
LastUpdated = newLastUpdated,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false);
}
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
{
// This connector relies on format-specific normalizers registered elsewhere.
throw new NotSupportedException("RedHatCsafConnector does not perform in-line normalization; use the CSAF normalizer component.");
}
private async Task<IReadOnlyList<RolieEntry>> FetchRolieEntriesAsync(Uri feedUri, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var ns = document.Root?.Name.Namespace ?? "http://www.w3.org/2005/Atom";
var entries = document.Root?
.Elements(ns + "entry")
.Select(e => new RolieEntry(
Id: (string?)e.Element(ns + "id"),
Updated: ParseUpdated((string?)e.Element(ns + "updated")),
DocumentUri: ParseDocumentLink(e, ns)))
.Where(entry => entry.Id is not null && entry.Updated is not null)
.OrderBy(entry => entry.Updated)
.ToList() ?? new List<RolieEntry>();
return entries;
}
private static DateTimeOffset? ParseUpdated(string? value)
=> DateTimeOffset.TryParse(value, out var parsed) ? parsed : null;
private static Uri? ParseDocumentLink(XElement entry, XNamespace ns)
{
var linkElements = entry.Elements(ns + "link");
foreach (var link in linkElements)
{
var rel = (string?)link.Attribute("rel");
var href = (string?)link.Attribute("href");
if (string.IsNullOrWhiteSpace(href))
{
continue;
}
if (rel is null || rel.Equals("enclosure", StringComparison.OrdinalIgnoreCase) || rel.Equals("alternate", StringComparison.OrdinalIgnoreCase))
{
if (Uri.TryCreate(href, UriKind.Absolute, out var uri))
{
return uri;
}
}
}
return null;
}
private async Task<VexRawDocument> DownloadCsafDocumentAsync(RolieEntry entry, CancellationToken cancellationToken)
{
var documentUri = entry.DocumentUri ?? throw new InvalidOperationException("ROLIE entry missing document URI.");
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
using var response = await client.GetAsync(documentUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(builder => builder
.Add("redhat.csaf.entryId", entry.Id)
.Add("redhat.csaf.documentUri", documentUri.ToString())
.Add("redhat.csaf.updated", entry.Updated?.ToString("O")));
return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, contentBytes, metadata);
}
private sealed record RolieEntry(string? Id, DateTimeOffset? Updated, Uri? DocumentUri);
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,10 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-RH-01-001 Provider metadata discovery|Team Excititor Connectors Red Hat|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added `RedHatProviderMetadataLoader` with HTTP/ETag caching, offline snapshot handling, and validation; exposed DI helper + tests covering live, cached, and offline scenarios.|
|EXCITITOR-CONN-RH-01-002 Incremental CSAF pulls|Team Excititor Connectors Red Hat|EXCITITOR-CONN-RH-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** Implemented `RedHatCsafConnector` with ROLIE feed parsing, incremental filtering via `context.Since`, CSAF document download + metadata capture, and persistence through `IVexRawDocumentSink`; tests cover live fetch/cache/offline scenarios with ETag handling.|
|EXCITITOR-CONN-RH-01-003 Trust metadata emission|Team Excititor Connectors Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-POLICY-01-001|**DONE (2025-10-17)** Provider metadata loader now emits trust overrides (weight, cosign issuer/pattern, PGP fingerprints) and the connector surfaces provenance hints for policy/consensus layers.|
|EXCITITOR-CONN-RH-01-004 Resume state persistence|Team Excititor Connectors Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** Connector now loads/saves resume state via `IVexConnectorStateRepository`, tracking last update timestamp and recent document digests to avoid duplicate CSAF ingestion; regression covers state persistence and duplicate skips.|
|EXCITITOR-CONN-RH-01-005 Worker/WebService integration|Team Excititor Connectors Red Hat|EXCITITOR-CONN-RH-01-002|**DONE (2025-10-17)** Worker/WebService now call `AddRedHatCsafConnector`, register the connector + state repo, and default worker scheduling adds the `excititor:redhat` provider so background jobs and orchestration can activate the connector without extra wiring.|
|EXCITITOR-CONN-RH-01-006 CSAF normalization parity tests|Team Excititor Connectors Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-FMT-CSAF-01-001|**DONE (2025-10-17)** Added RHSA fixture-driven regression verifying CSAF normalizer retains Red Hat product metadata, tracking fields, and timestamps (`rhsa-sample.json` + `CsafNormalizerTests.NormalizeAsync_PreservesRedHatSpecificMetadata`).|

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector targeting SUSE Rancher VEX Hub feeds, ingesting hub events and translating them into raw documents for normalization.
## Scope
- Hub discovery, authentication, and subscription handling for Rancher VEX updates.
- HTTP/WebSocket (if provided) ingestion, checkpoint tracking, and deduplication.
- Mapping Rancher-specific status fields and product identifiers into connector metadata.
- Integration with offline bundles to allow snapshot imports.
## Participants
- Worker manages scheduled syncs using this connector; WebService can trigger manual reconcile pulls.
- Normalizers convert retrieved documents via CSAF/OpenVEX workflows depending on payload.
- Policy module uses trust metadata produced here for weight evaluation.
## Interfaces & contracts
- Implements `IVexConnector` with options for hub URL, credentials, and poll intervals.
- Uses shared abstractions for resume markers and telemetry.
## In/Out of scope
In: hub connectivity, message processing, raw persistence, provider metadata.
Out: normalization/export tasks, storage layer implementation, attestation.
## Observability & security expectations
- Log subscription IDs, batch sizes, and checkpoint updates while redacting secrets.
- Emit metrics for messages processed, lag, and retries.
## Tests
- Connector harness tests with simulated hub responses will live in `../StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests`.

View File

@@ -0,0 +1,171 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
public sealed class RancherHubTokenProvider
{
private const string CachePrefix = "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Token";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
private readonly ILogger<RancherHubTokenProvider> _logger;
private readonly SemaphoreSlim _semaphore = new(1, 1);
public RancherHubTokenProvider(IHttpClientFactory httpClientFactory, IMemoryCache cache, ILogger<RancherHubTokenProvider> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async ValueTask<RancherHubAccessToken?> GetAccessTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (options.PreferOfflineSnapshot)
{
_logger.LogDebug("Skipping token request because PreferOfflineSnapshot is enabled.");
return null;
}
var hasCredentials = !string.IsNullOrWhiteSpace(options.ClientId) &&
!string.IsNullOrWhiteSpace(options.ClientSecret) &&
options.TokenEndpoint is not null;
if (!hasCredentials)
{
if (!options.AllowAnonymousDiscovery)
{
_logger.LogDebug("No Rancher hub credentials configured; proceeding without Authorization header.");
}
return null;
}
var cacheKey = $"{CachePrefix}:{options.ClientId}";
if (_cache.TryGetValue<RancherHubAccessToken>(cacheKey, out var cachedToken) && cachedToken is not null && !cachedToken.IsExpired())
{
return cachedToken;
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_cache.TryGetValue<RancherHubAccessToken>(cacheKey, out cachedToken) && cachedToken is not null && !cachedToken.IsExpired())
{
return cachedToken;
}
var token = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false);
if (token is not null)
{
var lifetime = token.ExpiresAt - DateTimeOffset.UtcNow;
if (lifetime <= TimeSpan.Zero)
{
lifetime = TimeSpan.FromMinutes(5);
}
var absoluteExpiration = lifetime > TimeSpan.FromSeconds(30)
? DateTimeOffset.UtcNow + lifetime - TimeSpan.FromSeconds(30)
: DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10);
_cache.Set(cacheKey, token, new MemoryCacheEntryOptions
{
AbsoluteExpiration = absoluteExpiration,
});
}
return token;
}
finally
{
_semaphore.Release();
}
}
private async Task<RancherHubAccessToken?> RequestTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken)
{
using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint);
request.Headers.Accept.ParseAdd("application/json");
var parameters = new Dictionary<string, string>
{
["grant_type"] = "client_credentials",
};
if (options.Scopes.Count > 0)
{
parameters["scope"] = string.Join(' ', options.Scopes);
}
if (!string.IsNullOrWhiteSpace(options.Audience))
{
parameters["audience"] = options.Audience!;
}
if (string.Equals(options.ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal))
{
var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.ClientId}:{options.ClientSecret}"));
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
}
else
{
parameters["client_id"] = options.ClientId!;
parameters["client_secret"] = options.ClientSecret!;
}
request.Content = new FormUrlEncodedContent(parameters);
var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException($"Failed to acquire Rancher hub access token ({response.StatusCode}): {payload}");
}
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (!root.TryGetProperty("access_token", out var accessTokenProperty) || accessTokenProperty.ValueKind is not JsonValueKind.String)
{
throw new InvalidOperationException("Token endpoint response missing access_token.");
}
var token = accessTokenProperty.GetString();
if (string.IsNullOrWhiteSpace(token))
{
throw new InvalidOperationException("Token endpoint response contained an empty access_token.");
}
var tokenType = root.TryGetProperty("token_type", out var tokenTypeElement) && tokenTypeElement.ValueKind is JsonValueKind.String
? tokenTypeElement.GetString() ?? "Bearer"
: "Bearer";
var expires = root.TryGetProperty("expires_in", out var expiresElement) &&
expiresElement.ValueKind is JsonValueKind.Number &&
expiresElement.TryGetInt32(out var expiresSeconds)
? DateTimeOffset.UtcNow + TimeSpan.FromSeconds(Math.Max(30, expiresSeconds))
: DateTimeOffset.UtcNow + TimeSpan.FromMinutes(30);
_logger.LogDebug("Acquired Rancher hub access token (expires {Expires}).", expires);
return new RancherHubAccessToken(token, tokenType, expires);
}
}
public sealed record RancherHubAccessToken(string Value, string TokenType, DateTimeOffset ExpiresAt)
{
public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt - TimeSpan.FromMinutes(1);
}

View File

@@ -0,0 +1,186 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
public sealed class RancherHubConnectorOptions
{
public static readonly Uri DefaultDiscoveryUri = new("https://vexhub.suse.com/.well-known/vex/rancher-hub.json");
/// <summary>
/// HTTP client name registered for the connector.
/// </summary>
public const string HttpClientName = "excititor.connector.suse.rancherhub";
/// <summary>
/// URI for the Rancher VEX hub discovery document.
/// </summary>
public Uri DiscoveryUri { get; set; } = DefaultDiscoveryUri;
/// <summary>
/// Optional OAuth2/OIDC token endpoint used for hub authentication.
/// </summary>
public Uri? TokenEndpoint { get; set; }
/// <summary>
/// Client identifier used when requesting hub access tokens.
/// </summary>
public string? ClientId { get; set; }
/// <summary>
/// Client secret used when requesting hub access tokens.
/// </summary>
public string? ClientSecret { get; set; }
/// <summary>
/// OAuth scopes requested for hub access; defaults align with Rancher hub reader role.
/// </summary>
public IList<string> Scopes { get; } = new List<string> { "hub.read" };
/// <summary>
/// Optional audience claim passed when requesting tokens (client credential grant).
/// </summary>
public string? Audience { get; set; }
/// <summary>
/// Preferred authentication scheme. Supported: client_secret_basic (default) or client_secret_post.
/// </summary>
public string ClientAuthenticationScheme { get; set; } = "client_secret_basic";
/// <summary>
/// Duration to cache discovery metadata before re-fetching.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromMinutes(30);
/// <summary>
/// Optional file path for discovery metadata snapshots.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// When true, the loader prefers the offline snapshot prior to attempting network discovery.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Enables persisting freshly fetched discovery documents to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
/// <summary>
/// Weight applied to the provider entry; hubs default below direct vendor feeds.
/// </summary>
public double TrustWeight { get; set; } = 0.6;
/// <summary>
/// Optional Sigstore/Cosign issuer for verifying hub-delivered attestations.
/// </summary>
public string? CosignIssuer { get; set; }
/// <summary>
/// Cosign identity pattern matched against transparency log subjects.
/// </summary>
public string? CosignIdentityPattern { get; set; }
/// <summary>
/// Additional trusted PGP fingerprints declared by the hub.
/// </summary>
public IList<string> PgpFingerprints { get; } = new List<string>();
/// <summary>
/// Allows falling back to unauthenticated discovery requests when credentials are absent.
/// </summary>
public bool AllowAnonymousDiscovery { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (DiscoveryUri is null || !DiscoveryUri.IsAbsoluteUri)
{
throw new InvalidOperationException("DiscoveryUri must be an absolute URI.");
}
if (DiscoveryUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("DiscoveryUri must use HTTP or HTTPS.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("MetadataCacheDuration must be positive.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled.");
}
var hasClientId = !string.IsNullOrWhiteSpace(ClientId);
var hasClientSecret = !string.IsNullOrWhiteSpace(ClientSecret);
var hasTokenEndpoint = TokenEndpoint is not null;
if (hasClientId || hasClientSecret || hasTokenEndpoint)
{
if (!(hasClientId && hasClientSecret && hasTokenEndpoint))
{
throw new InvalidOperationException("ClientId, ClientSecret, and TokenEndpoint must be provided together for authenticated discovery.");
}
if (TokenEndpoint is not null && (!TokenEndpoint.IsAbsoluteUri || TokenEndpoint.Scheme is not ("http" or "https")))
{
throw new InvalidOperationException("TokenEndpoint must be an absolute HTTP(S) URI.");
}
}
if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight))
{
TrustWeight = 0.6;
}
else if (TrustWeight <= 0)
{
TrustWeight = 0.1;
}
else if (TrustWeight > 1.0)
{
TrustWeight = 1.0;
}
if (!string.IsNullOrWhiteSpace(CosignIssuer) && string.IsNullOrWhiteSpace(CosignIdentityPattern))
{
throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified.");
}
if (!string.Equals(ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal) &&
!string.Equals(ClientAuthenticationScheme, "client_secret_post", StringComparison.Ordinal))
{
throw new InvalidOperationException("ClientAuthenticationScheme must be 'client_secret_basic' or 'client_secret_post'.");
}
// Remove any empty scopes to avoid token request issues.
if (Scopes.Count > 0)
{
for (var i = Scopes.Count - 1; i >= 0; i--)
{
if (string.IsNullOrWhiteSpace(Scopes[i]))
{
Scopes.RemoveAt(i);
}
}
}
if (Scopes.Count == 0)
{
Scopes.Add("hub.read");
}
}
}

View File

@@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
public sealed class RancherHubConnectorOptionsValidator : IVexConnectorOptionsValidator<RancherHubConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public RancherHubConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(VexConnectorDescriptor descriptor, RancherHubConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,49 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.DependencyInjection;
public static class RancherHubConnectorServiceCollectionExtensions
{
public static IServiceCollection AddRancherHubConnector(this IServiceCollection services, Action<RancherHubConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<RancherHubConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
});
services.AddSingleton<IVexConnectorOptionsValidator<RancherHubConnectorOptions>, RancherHubConnectorOptionsValidator>();
services.AddSingleton<RancherHubTokenProvider>();
services.AddSingleton<RancherHubMetadataLoader>();
services.AddSingleton<IVexConnector, RancherHubConnector>();
services.AddHttpClient(RancherHubConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
return services;
}
}

View File

@@ -0,0 +1,127 @@
# EXCITITOR-CONN-SUSE-01-002 — Checkpointed Event Ingestion (Design)
**Status:** draft • **Updated:** 2025-10-19
**Scope:** StellaOps.Excititor.Connectors.SUSE.RancherVEXHub
## Goals
- Stream Rancher VEX Hub events deterministically, supporting cold start and incremental resumes.
- Persist checkpoints so subsequent runs (worker/manual CLI) resume where the previous execution stopped.
- Deduplicate hub payloads using cryptographic digests while keeping a short history (≤ 200 entries) to align with `IVexConnectorStateRepository` constraints.
- Quarantine malformed/unfetchable events without blocking healthy ones, making failures observable for operators.
- Remain offline-friendly: work from discovery metadata snapshots and cached checkpoints without live network calls when configured.
## Assumed Event Model
Discovery metadata supplies `subscription.eventsUri` and (optionally) `subscription.checkpointUri`. Rancher emits JSON event batches over HTTP(S):
```json
{
"cursor": "opaque-offset-123",
"events": [
{
"id": "evt-2025-10-19T12:42:30Z-001",
"type": "vex.statement.published",
"channel": "rancher/rke2",
"publishedAt": "2025-10-19T12:42:30Z",
"document": {
"uri": "https://hub.suse.example/events/evt-.../statement.json",
"sha256": "ab12...",
"format": "csaf"
}
}
]
}
```
Key properties assumed per discovery schema validation:
- `cursor` advances monotonically and can be replayed via `?cursor=<value>` or a POST to `checkpointUri`.
- Events carry a `document.uri` (absolute HTTP(S) URI) and an optional digest (`document.sha256`). When absent, a digest is computed after download.
- `publishedAt` is UTC and stable; it is used as `VexConnectorState.LastUpdated` fallback when no checkpoint is provided.
- Optional `channels` allow filtering (`channels=foo,bar`) to minimise payloads.
The connector must tolerate missing fields by quaratining the raw envelope.
## Flow Overview
1. **Load connector state** from `IVexConnectorStateRepository` keyed by `Descriptor.Id`.
- `LastUpdated` stores the last successfully processed `publishedAt`.
- `DocumentDigests` stores:
- Last checkpoint token entry prefixed `checkpoint:` (only most recent kept).
- Recent raw document digests for deduping.
2. **Resolve resume parameters**:
- Start cursor: explicit CLI `context.Since` overrides persisted checkpoint.
- If checkpoint exists, call `eventsUri?cursor=<encoded>`; else pass `since=<ISO-8601>` (from `state.LastUpdated` or `context.Since`).
- Limit channels if discovery enumerated them and options specify `RancherHubConnectorOptions.EnabledChannels` (future option).
3. **Fetch batches** in a deterministic, cancellation-aware loop:
- Send GETs with `pageSize` cap (default 200) and follow `nextCursor`/pagination until exhaustion.
- For each batch log metrics (`eventCount`, `cursor`, `fromOffline` flag).
4. **Process events**:
- Validate minimal shape (id, document uri). Missing/invalid fields => log warning + quarantine JSON payload.
- Fetch document content via shared HTTP client. Respect optional digests (compare after download).
- Build raw metadata: event ID, channel, publishedAt, checkpoint cursor (if provided), offline flag.
- Deduplicate using `HashSet` seeded with persisted digests; skip duplicates without re-writing state.
- Push valid documents to `context.RawSink.StoreAsync` and yield them downstream.
- Capture latest `publishedAt` and `cursor` for state update.
5. **Quarantine path**:
- Serialize offending envelope into UTF-8 JSON (`application/vnd.stella.quarantine+json` metadata flag).
- Persist via `context.RawSink.StoreAsync` using format `VexDocumentFormat.Json` and metadata `{"rancher.event.quarantine":"true"}` to allow downstream filtering/reporting.
6. **Persist state** once the batch completes or on graceful cancellation:
- Update `LastUpdated` with max `publishedAt` processed.
- Rebuild digest window (most recent ≤ 200).
- Store latest checkpoint token (if hub supplied one) as first digest entry `checkpoint:<base64url>` for quick retrieval.
## Key Types & Components
```csharp
internal sealed record RancherHubEventEnvelope(
string Id,
string? Type,
string Channel,
DateTimeOffset PublishedAt,
Uri DocumentUri,
string? DocumentDigest,
string? DocumentFormat);
internal sealed record RancherHubCheckpointState(
string? Cursor,
DateTimeOffset? LatestPublishedAt,
ImmutableArray<string> Digests);
```
- `RancherHubEventClient` (new) encapsulates HTTP paging, cursor handling, and offline replay (reading bundled snapshot JSON when `PreferOfflineSnapshot` enabled).
- `RancherHubCheckpointManager` (new) reads/writes `VexConnectorState`, encoding checkpoint token under the `checkpoint:` prefix and trimming digest history.
## Deduplication Strategy
- Primary key: document SHA-256 digest (hub-provided or computed). Fallback: `event.Id` when digest missing (encoded as `event:<id>` entry).
- Persist dedupe keys via `DocumentDigests` to short-circuit duplicates on next run. Keep insertion order for deterministic state updates.
- When offline snapshot is replayed, skip dedupe reset—reused digests still apply.
## Quarantine Semantics
- Trigger conditions:
- JSON envelope missing required fields.
- Document fetch returns non-success HTTP code.
- Digest mismatch between declared `document.sha256` and computed value.
- Action: create `VexRawDocument` with metadata:
- `rancher.event.id`, `rancher.event.channel`, `rancher.event.type`, `rancher.event.error`.
- `rancher.event.quarantine=true` flag for downstream routing.
- Content: original envelope JSON (or error stub when fetch failed).
- Quarantine entries count toward dedupe history using a synthetic digest `quarantine:<eventId>` to prevent repeated attempts until manual intervention.
## Cancellation & Determinism
- Each HTTP call honours `CancellationToken`.
- Loop checkpoints after each processed batch; if cancellation occurs mid-batch, state updates only include successfully handled documents to preserve deterministic replays.
- Sorting: events processed in ascending `publishedAt` (or server-provided order). Within batch, maintain original order to avoid digest reshuffling.
## Open Questions / Follow-ups
- Confirm exact Rancher event schema (pending coordination with SUSE PSIRT) and adjust parser accordingly.
- Validate whether `checkpointUri` requires POST with body `{ "cursor": "..."} ` or simple GET.
- Decide on channel filtering surface area (option flag vs. discovery default).
- Establish metrics contract once observability task (future) starts.
Until those are resolved the implementation will keep parser tolerant with detailed logging and quarantine coverage so future adjustments are low risk.

View File

@@ -0,0 +1,311 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
public sealed class RancherHubEventClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly RancherHubTokenProvider _tokenProvider;
private readonly IFileSystem _fileSystem;
private readonly ILogger<RancherHubEventClient> _logger;
private readonly JsonDocumentOptions _documentOptions = new()
{
CommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
};
private const string CheckpointPrefix = "checkpoint";
public RancherHubEventClient(
IHttpClientFactory httpClientFactory,
RancherHubTokenProvider tokenProvider,
IFileSystem fileSystem,
ILogger<RancherHubEventClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async IAsyncEnumerable<RancherHubEventBatch> FetchEventBatchesAsync(
RancherHubConnectorOptions options,
RancherHubMetadata metadata,
string? cursor,
DateTimeOffset? since,
ImmutableArray<string> channels,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(metadata);
if (options.PreferOfflineSnapshot && metadata.OfflineSnapshot is not null)
{
var offline = await LoadOfflineSnapshotAsync(metadata.OfflineSnapshot, cancellationToken).ConfigureAwait(false);
if (offline is not null)
{
yield return offline;
}
yield break;
}
var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
var currentCursor = cursor;
var currentSince = since;
var firstRequest = true;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var requestUri = BuildRequestUri(metadata.Subscription.EventsUri, currentCursor, currentSince, channels);
using var request = await CreateRequestAsync(options, metadata, requestUri, cancellationToken).ConfigureAwait(false);
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException($"Rancher hub events request failed ({(int)response.StatusCode} {response.StatusCode}). Payload: {payload}");
}
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var batch = ParseBatch(json, fromOfflineSnapshot: false);
yield return batch;
if (string.IsNullOrWhiteSpace(batch.NextCursor))
{
break;
}
if (!firstRequest && string.Equals(batch.NextCursor, currentCursor, StringComparison.Ordinal))
{
_logger.LogWarning("Detected stable cursor {Cursor}; stopping to avoid loop.", batch.NextCursor);
break;
}
currentCursor = batch.NextCursor;
currentSince = null; // cursor supersedes since parameter
firstRequest = false;
}
}
private async Task<RancherHubEventBatch?> LoadOfflineSnapshotAsync(RancherHubOfflineSnapshotMetadata offline, CancellationToken cancellationToken)
{
try
{
string payload;
if (offline.SnapshotUri.Scheme.Equals("file", StringComparison.OrdinalIgnoreCase))
{
var path = offline.SnapshotUri.LocalPath;
payload = await _fileSystem.File.ReadAllTextAsync(path, Encoding.UTF8, cancellationToken).ConfigureAwait(false);
}
else
{
var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
using var response = await client.GetAsync(offline.SnapshotUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
}
if (!string.IsNullOrWhiteSpace(offline.Sha256))
{
var computed = ComputeSha256(payload);
if (!string.Equals(computed, offline.Sha256, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Offline snapshot digest mismatch (expected {Expected}, computed {Computed}); proceeding anyway.",
offline.Sha256,
computed);
}
}
return ParseBatch(payload, fromOfflineSnapshot: true);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to load Rancher hub offline snapshot from {Uri}.", offline.SnapshotUri);
return null;
}
}
private async Task<HttpRequestMessage> CreateRequestAsync(RancherHubConnectorOptions options, RancherHubMetadata metadata, Uri requestUri, CancellationToken cancellationToken)
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Accept.ParseAdd("application/json");
if (metadata.Subscription.RequiresAuthentication)
{
var token = await _tokenProvider.GetAccessTokenAsync(options, cancellationToken).ConfigureAwait(false);
if (token is not null)
{
var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType;
request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value);
}
}
return request;
}
private RancherHubEventBatch ParseBatch(string payload, bool fromOfflineSnapshot)
{
using var document = JsonDocument.Parse(payload, _documentOptions);
var root = document.RootElement;
var cursor = ReadString(root, "cursor", "currentCursor", "checkpoint");
var nextCursor = ReadString(root, "nextCursor", "next", "continuation", "continuationToken");
var eventsElement = TryGetProperty(root, "events", "items", "data") ?? default;
var events = ImmutableArray.CreateBuilder<RancherHubEventRecord>();
if (eventsElement.ValueKind == JsonValueKind.Array)
{
foreach (var item in eventsElement.EnumerateArray())
{
events.Add(ParseEvent(item));
}
}
return new RancherHubEventBatch(cursor, nextCursor, events.ToImmutable(), fromOfflineSnapshot, payload);
}
private RancherHubEventRecord ParseEvent(JsonElement element)
{
var rawJson = element.GetRawText();
var id = ReadString(element, "id", "eventId", "uuid");
var type = ReadString(element, "type", "eventType");
var channel = ReadString(element, "channel", "product", "stream");
var publishedAt = ParseDate(ReadString(element, "publishedAt", "timestamp", "createdAt"));
Uri? documentUri = null;
string? documentDigest = null;
string? documentFormat = null;
var documentElement = TryGetProperty(element, "document", "payload", "statement");
if (documentElement.HasValue)
{
documentUri = ParseUri(ReadString(documentElement.Value, "uri", "url", "href"));
documentDigest = ReadString(documentElement.Value, "sha256", "digest", "checksum");
documentFormat = ReadString(documentElement.Value, "format", "kind", "type");
}
else
{
documentUri = ParseUri(ReadString(element, "documentUri", "uri", "url"));
documentDigest = ReadString(element, "documentSha256", "sha256", "digest");
documentFormat = ReadString(element, "documentFormat", "format");
}
return new RancherHubEventRecord(rawJson, id, type, channel, publishedAt, documentUri, documentDigest, documentFormat);
}
private static Uri? ParseUri(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null;
private static DateTimeOffset? ParseDate(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : DateTimeOffset.TryParse(value, out var parsed) ? parsed : null;
private static string? ReadString(JsonElement element, params string[] propertyNames)
{
var property = TryGetProperty(element, propertyNames);
if (!property.HasValue || property.Value.ValueKind is not JsonValueKind.String)
{
return null;
}
var value = property.Value.GetString();
return string.IsNullOrWhiteSpace(value) ? null : value;
}
private static JsonElement? TryGetProperty(JsonElement element, params string[] propertyNames)
{
foreach (var propertyName in propertyNames)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind is not JsonValueKind.Null and not JsonValueKind.Undefined)
{
return property;
}
}
return null;
}
private static string BuildQueryString(Dictionary<string, string> parameters)
{
if (parameters.Count == 0)
{
return string.Empty;
}
var builder = new StringBuilder();
var first = true;
foreach (var kvp in parameters)
{
if (string.IsNullOrEmpty(kvp.Value))
{
continue;
}
if (!first)
{
builder.Append('&');
}
builder.Append(Uri.EscapeDataString(kvp.Key));
builder.Append('=');
builder.Append(Uri.EscapeDataString(kvp.Value));
first = false;
}
return builder.ToString();
}
private static Uri BuildRequestUri(Uri baseUri, string? cursor, DateTimeOffset? since, ImmutableArray<string> channels)
{
var builder = new UriBuilder(baseUri);
var parameters = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(cursor))
{
parameters["cursor"] = cursor;
}
else if (since is not null)
{
parameters["since"] = since.Value.ToUniversalTime().ToString("O");
}
if (!channels.IsDefaultOrEmpty && channels.Length > 0)
{
parameters["channels"] = string.Join(',', channels);
}
var query = BuildQueryString(parameters);
builder.Query = string.IsNullOrEmpty(query) ? null : query;
return builder.Uri;
}
private static string ComputeSha256(string payload)
{
var bytes = Encoding.UTF8.GetBytes(payload);
Span<byte> hash = stackalloc byte[32];
if (SHA256.TryHashData(bytes, hash, out _))
{
return Convert.ToHexString(hash).ToLowerInvariant();
}
using var sha = SHA256.Create();
return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant();
}
}

View File

@@ -0,0 +1,21 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
public sealed record RancherHubEventRecord(
string RawJson,
string? Id,
string? Type,
string? Channel,
DateTimeOffset? PublishedAt,
Uri? DocumentUri,
string? DocumentDigest,
string? DocumentFormat);
public sealed record RancherHubEventBatch(
string? Cursor,
string? NextCursor,
ImmutableArray<RancherHubEventRecord> Events,
bool FromOfflineSnapshot,
string RawPayload);

View File

@@ -0,0 +1,455 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
public sealed class RancherHubMetadataLoader
{
public const string CachePrefix = "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly RancherHubTokenProvider _tokenProvider;
private readonly IFileSystem _fileSystem;
private readonly ILogger<RancherHubMetadataLoader> _logger;
private readonly SemaphoreSlim _semaphore = new(1, 1);
private readonly JsonDocumentOptions _documentOptions;
public RancherHubMetadataLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
RancherHubTokenProvider tokenProvider,
IFileSystem fileSystem,
ILogger<RancherHubMetadataLoader> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_documentOptions = new JsonDocumentOptions
{
CommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
};
}
public async Task<RancherHubMetadataResult> LoadAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out var cached) && cached is not null && !cached.IsExpired())
{
_logger.LogDebug("Returning cached Rancher hub metadata (expires {Expires}).", cached.ExpiresAt);
return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out cached) && cached is not null && !cached.IsExpired())
{
return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot);
}
CacheEntry? previous = cached;
CacheEntry? entry = null;
if (options.PreferOfflineSnapshot)
{
entry = TryLoadFromOffline(options);
if (entry is null)
{
throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline discovery snapshot was found.");
}
}
else
{
entry = await TryFetchFromNetworkAsync(options, previous, cancellationToken).ConfigureAwait(false);
if (entry is null)
{
entry = TryLoadFromOffline(options);
}
}
if (entry is null)
{
throw new InvalidOperationException("Unable to load Rancher hub discovery metadata from network or offline snapshot.");
}
_memoryCache.Set(cacheKey, entry, new MemoryCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = options.MetadataCacheDuration,
});
return new RancherHubMetadataResult(entry.Metadata, entry.FetchedAt, FromCache: false, entry.FromOfflineSnapshot);
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(RancherHubConnectorOptions options, CacheEntry? previous, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
using var request = new HttpRequestMessage(HttpMethod.Get, options.DiscoveryUri);
request.Headers.Accept.ParseAdd("application/json");
if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag))
{
request.Headers.IfNoneMatch.Add(etag);
}
var token = await _tokenProvider.GetAccessTokenAsync(options, cancellationToken).ConfigureAwait(false);
if (token is not null)
{
var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType;
request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value);
}
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotModified && previous is not null)
{
_logger.LogDebug("Rancher hub discovery document not modified (etag {ETag}).", previous.ETag);
return previous with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + options.MetadataCacheDuration,
FromOfflineSnapshot = false,
};
}
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var metadata = ParseMetadata(payload, options);
var entry = new CacheEntry(
metadata,
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow + options.MetadataCacheDuration,
response.Headers.ETag?.ToString(),
FromOfflineSnapshot: false,
Payload: payload);
PersistOfflineSnapshot(options, payload);
return entry;
}
catch (Exception ex) when (ex is not OperationCanceledException && !options.PreferOfflineSnapshot)
{
_logger.LogWarning(ex, "Failed to fetch Rancher hub discovery document; attempting offline snapshot fallback.");
return null;
}
}
private CacheEntry? TryLoadFromOffline(RancherHubConnectorOptions options)
{
if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(options.OfflineSnapshotPath))
{
_logger.LogWarning("Rancher hub offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath);
var metadata = ParseMetadata(payload, options);
return new CacheEntry(
metadata,
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow + options.MetadataCacheDuration,
ETag: null,
FromOfflineSnapshot: true,
Payload: payload);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Rancher hub discovery metadata from offline snapshot {Path}.", options.OfflineSnapshotPath);
return null;
}
}
private void PersistOfflineSnapshot(RancherHubConnectorOptions options, string payload)
{
if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return;
}
try
{
var directory = _fileSystem.Path.GetDirectoryName(options.OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory))
{
_fileSystem.Directory.CreateDirectory(directory);
}
_fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath);
}
}
private RancherHubMetadata ParseMetadata(string payload, RancherHubConnectorOptions options)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Rancher hub discovery payload was empty.");
}
try
{
using var document = JsonDocument.Parse(payload, _documentOptions);
var root = document.RootElement;
var hubId = ReadString(root, "hubId") ?? "excititor:suse:rancher";
var title = ReadString(root, "title") ?? ReadString(root, "displayName") ?? "SUSE Rancher VEX Hub";
var baseUri = ReadUri(root, "baseUri");
var subscriptionElement = TryGetProperty(root, "subscription");
if (!subscriptionElement.HasValue)
{
throw new InvalidOperationException("Discovery payload missing subscription section.");
}
var subscription = subscriptionElement.Value;
var eventsUri = ReadRequiredUri(subscription, "eventsUri", "eventsUrl", "eventsEndpoint");
var checkpointUri = ReadUri(subscription, "checkpointUri", "checkpointUrl", "checkpointEndpoint");
var channels = ReadStringArray(subscription, "channels", "defaultChannels", "products");
var scopes = ReadStringArray(subscription, "scopes", "defaultScopes");
var requiresAuth = ReadBoolean(subscription, "requiresAuthentication", defaultValue: options.TokenEndpoint is not null);
var authenticationElement = TryGetProperty(root, "authentication");
var tokenEndpointFromMetadata = authenticationElement.HasValue
? ReadUri(authenticationElement.Value, "tokenUri", "tokenEndpoint") ?? options.TokenEndpoint
: options.TokenEndpoint;
var audience = authenticationElement.HasValue
? ReadString(authenticationElement.Value, "audience", "aud") ?? options.Audience
: options.Audience;
var offlineElement = TryGetProperty(root, "offline", "snapshot");
var offlineSnapshot = offlineElement.HasValue
? BuildOfflineSnapshot(offlineElement.Value, options)
: null;
var provider = BuildProvider(hubId, title, baseUri, eventsUri, options);
var subscriptionMetadata = new RancherHubSubscriptionMetadata(eventsUri, checkpointUri, channels, scopes, requiresAuth);
var authenticationMetadata = new RancherHubAuthenticationMetadata(tokenEndpointFromMetadata, audience);
return new RancherHubMetadata(provider, subscriptionMetadata, authenticationMetadata, offlineSnapshot);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Failed to parse Rancher hub discovery payload.", ex);
}
}
private RancherHubOfflineSnapshotMetadata? BuildOfflineSnapshot(JsonElement element, RancherHubConnectorOptions options)
{
var snapshotUri = ReadUri(element, "snapshotUri", "uri", "url");
if (snapshotUri is null)
{
return null;
}
var checksum = ReadString(element, "sha256", "checksum", "digest");
DateTimeOffset? updatedAt = null;
var updatedString = ReadString(element, "updated", "lastModified", "timestamp");
if (!string.IsNullOrWhiteSpace(updatedString) && DateTimeOffset.TryParse(updatedString, out var parsed))
{
updatedAt = parsed;
}
return new RancherHubOfflineSnapshotMetadata(snapshotUri, checksum, updatedAt);
}
private VexProvider BuildProvider(string hubId, string title, Uri? baseUri, Uri eventsUri, RancherHubConnectorOptions options)
{
var baseUris = new List<Uri>();
if (baseUri is not null)
{
baseUris.Add(baseUri);
}
baseUris.Add(eventsUri);
VexCosignTrust? cosign = null;
if (!string.IsNullOrWhiteSpace(options.CosignIssuer) && !string.IsNullOrWhiteSpace(options.CosignIdentityPattern))
{
cosign = new VexCosignTrust(options.CosignIssuer!, options.CosignIdentityPattern!);
}
var trust = new VexProviderTrust(options.TrustWeight, cosign, options.PgpFingerprints);
return new VexProvider(hubId, title, VexProviderKind.Hub, baseUris, new VexProviderDiscovery(options.DiscoveryUri, null), trust);
}
private static string CreateCacheKey(RancherHubConnectorOptions options)
=> $"{CachePrefix}:{options.DiscoveryUri}";
private static JsonElement? TryGetProperty(JsonElement element, params string[] propertyNames)
{
foreach (var name in propertyNames)
{
if (element.TryGetProperty(name, out var value) && value.ValueKind is not JsonValueKind.Null and not JsonValueKind.Undefined)
{
return value;
}
}
return null;
}
private static string? ReadString(JsonElement element, params string[] propertyNames)
{
var property = TryGetProperty(element, propertyNames);
if (property is null || property.Value.ValueKind is not JsonValueKind.String)
{
return null;
}
var value = property.Value.GetString();
return string.IsNullOrWhiteSpace(value) ? null : value;
}
private static bool ReadBoolean(JsonElement element, string propertyName, bool defaultValue)
{
if (!element.TryGetProperty(propertyName, out var property))
{
return defaultValue;
}
return property.ValueKind switch
{
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed,
_ => defaultValue,
};
}
private static ImmutableArray<string> ReadStringArray(JsonElement element, params string[] propertyNames)
{
var property = TryGetProperty(element, propertyNames);
if (property is null)
{
return ImmutableArray<string>.Empty;
}
if (property.Value.ValueKind is JsonValueKind.Array)
{
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var item in property.Value.EnumerateArray())
{
if (item.ValueKind is JsonValueKind.String)
{
var value = item.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value!);
}
}
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
if (property.Value.ValueKind is JsonValueKind.String)
{
var single = property.Value.GetString();
return string.IsNullOrWhiteSpace(single)
? ImmutableArray<string>.Empty
: ImmutableArray.Create(single!);
}
return ImmutableArray<string>.Empty;
}
private static Uri? ReadUri(JsonElement element, params string[] propertyNames)
{
var value = ReadString(element, propertyNames);
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException($"Discovery field '{string.Join("/", propertyNames)}' must be an absolute HTTP(S) URI.");
}
return uri;
}
private static Uri ReadRequiredUri(JsonElement element, params string[] propertyNames)
{
var uri = ReadUri(element, propertyNames);
if (uri is null)
{
throw new InvalidOperationException($"Discovery payload missing required URI field '{string.Join("/", propertyNames)}'.");
}
return uri;
}
private sealed record CacheEntry(
RancherHubMetadata Metadata,
DateTimeOffset FetchedAt,
DateTimeOffset ExpiresAt,
string? ETag,
bool FromOfflineSnapshot,
string? Payload)
{
public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt;
}
}
public sealed record RancherHubMetadata(
VexProvider Provider,
RancherHubSubscriptionMetadata Subscription,
RancherHubAuthenticationMetadata Authentication,
RancherHubOfflineSnapshotMetadata? OfflineSnapshot);
public sealed record RancherHubSubscriptionMetadata(
Uri EventsUri,
Uri? CheckpointUri,
ImmutableArray<string> Channels,
ImmutableArray<string> Scopes,
bool RequiresAuthentication);
public sealed record RancherHubAuthenticationMetadata(
Uri? TokenEndpoint,
string? Audience);
public sealed record RancherHubOfflineSnapshotMetadata(
Uri SnapshotUri,
string? Sha256,
DateTimeOffset? UpdatedAt);
public sealed record RancherHubMetadataResult(
RancherHubMetadata Metadata,
DateTimeOffset FetchedAt,
bool FromCache,
bool FromOfflineSnapshot);

View File

@@ -0,0 +1,345 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub;
public sealed class RancherHubConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor StaticDescriptor = new(
id: "excititor:suse.rancher",
kind: VexProviderKind.Hub,
displayName: "SUSE Rancher VEX Hub")
{
Tags = ImmutableArray.Create("hub", "suse", "offline"),
};
private readonly RancherHubMetadataLoader _metadataLoader;
private readonly RancherHubEventClient _eventClient;
private readonly RancherHubCheckpointManager _checkpointManager;
private readonly RancherHubTokenProvider _tokenProvider;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IEnumerable<IVexConnectorOptionsValidator<RancherHubConnectorOptions>> _validators;
private RancherHubConnectorOptions? _options;
private RancherHubMetadataResult? _metadata;
public RancherHubConnector(
RancherHubMetadataLoader metadataLoader,
RancherHubEventClient eventClient,
RancherHubCheckpointManager checkpointManager,
RancherHubTokenProvider tokenProvider,
IHttpClientFactory httpClientFactory,
ILogger<RancherHubConnector> logger,
TimeProvider timeProvider,
IEnumerable<IVexConnectorOptionsValidator<RancherHubConnectorOptions>>? validators = null)
: base(StaticDescriptor, logger, timeProvider)
{
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
_eventClient = eventClient ?? throw new ArgumentNullException(nameof(eventClient));
_checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<RancherHubConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary<string, object?>
{
["discoveryUri"] = _options.DiscoveryUri.ToString(),
["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(),
["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication,
["fromOffline"] = _metadata.FromOfflineSnapshot,
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_metadata is null)
{
_metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
}
var checkpoint = await _checkpointManager.LoadAsync(Descriptor.Id, context, cancellationToken).ConfigureAwait(false);
var digestHistory = checkpoint.Digests.ToList();
var dedupeSet = new HashSet<string>(checkpoint.Digests, StringComparer.OrdinalIgnoreCase);
var latestCursor = checkpoint.Cursor;
var latestPublishedAt = checkpoint.LastPublishedAt ?? checkpoint.EffectiveSince;
var stateChanged = false;
LogConnectorEvent(LogLevel.Information, "fetch_start", "Starting Rancher hub event ingestion.", new Dictionary<string, object?>
{
["since"] = checkpoint.EffectiveSince?.ToString("O"),
["cursor"] = checkpoint.Cursor,
["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(),
["offline"] = checkpoint.Cursor is null && _options.PreferOfflineSnapshot,
});
await foreach (var batch in _eventClient.FetchEventBatchesAsync(
_options,
_metadata.Metadata,
checkpoint.Cursor,
checkpoint.EffectiveSince,
_metadata.Metadata.Subscription.Channels,
cancellationToken).ConfigureAwait(false))
{
LogConnectorEvent(LogLevel.Debug, "batch", "Processing Rancher hub batch.", new Dictionary<string, object?>
{
["cursor"] = batch.Cursor,
["nextCursor"] = batch.NextCursor,
["count"] = batch.Events.Length,
["offline"] = batch.FromOfflineSnapshot,
});
if (!string.IsNullOrWhiteSpace(batch.NextCursor) && !string.Equals(batch.NextCursor, latestCursor, StringComparison.Ordinal))
{
latestCursor = batch.NextCursor;
stateChanged = true;
}
else if (string.IsNullOrWhiteSpace(latestCursor) && !string.IsNullOrWhiteSpace(batch.Cursor))
{
latestCursor = batch.Cursor;
}
foreach (var record in batch.Events)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await ProcessEventAsync(record, batch, context, dedupeSet, digestHistory, cancellationToken).ConfigureAwait(false);
if (result.ProcessedDocument is not null)
{
yield return result.ProcessedDocument;
stateChanged = true;
if (result.PublishedAt is { } published && (latestPublishedAt is null || published > latestPublishedAt))
{
latestPublishedAt = published;
}
}
else if (result.Quarantined)
{
stateChanged = true;
}
}
}
if (stateChanged || !string.Equals(latestCursor, checkpoint.Cursor, StringComparison.Ordinal) || latestPublishedAt != checkpoint.LastPublishedAt)
{
await _checkpointManager.SaveAsync(
Descriptor.Id,
latestCursor,
latestPublishedAt,
digestHistory.ToImmutableArray(),
cancellationToken).ConfigureAwait(false);
}
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads.");
public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata;
private async Task<EventProcessingResult> ProcessEventAsync(
RancherHubEventRecord record,
RancherHubEventBatch batch,
VexConnectorContext context,
HashSet<string> dedupeSet,
List<string> digestHistory,
CancellationToken cancellationToken)
{
var quarantineKey = BuildQuarantineKey(record);
if (dedupeSet.Contains(quarantineKey))
{
return EventProcessingResult.QuarantinedOnly;
}
if (record.DocumentUri is null || string.IsNullOrWhiteSpace(record.Id))
{
await QuarantineAsync(record, batch, "missing documentUri or id", context, cancellationToken).ConfigureAwait(false);
AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory);
return EventProcessingResult.QuarantinedOnly;
}
var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
using var request = await CreateDocumentRequestAsync(record.DocumentUri, cancellationToken).ConfigureAwait(false);
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
await QuarantineAsync(record, batch, $"document fetch failed ({(int)response.StatusCode} {response.StatusCode})", context, cancellationToken).ConfigureAwait(false);
AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory);
return EventProcessingResult.QuarantinedOnly;
}
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var publishedAt = record.PublishedAt ?? UtcNow();
var metadata = BuildMetadata(builder => builder
.Add("rancher.event.id", record.Id)
.Add("rancher.event.type", record.Type)
.Add("rancher.event.channel", record.Channel)
.Add("rancher.event.published", publishedAt)
.Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor)
.Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false")
.Add("rancher.event.declaredDigest", record.DocumentDigest));
var format = ResolveFormat(record.DocumentFormat);
var document = CreateRawDocument(format, record.DocumentUri, contentBytes, metadata);
if (!string.IsNullOrWhiteSpace(record.DocumentDigest))
{
var declared = NormalizeDigest(record.DocumentDigest);
var computed = NormalizeDigest(document.Digest);
if (!string.Equals(declared, computed, StringComparison.OrdinalIgnoreCase))
{
await QuarantineAsync(record, batch, $"digest mismatch (declared {record.DocumentDigest}, computed {document.Digest})", context, cancellationToken).ConfigureAwait(false);
AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory);
return EventProcessingResult.QuarantinedOnly;
}
}
if (!dedupeSet.Add(document.Digest))
{
return EventProcessingResult.Skipped;
}
digestHistory.Add(document.Digest);
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
return new EventProcessingResult(document, false, publishedAt);
}
private async Task<HttpRequestMessage> CreateDocumentRequestAsync(Uri documentUri, CancellationToken cancellationToken)
{
var request = new HttpRequestMessage(HttpMethod.Get, documentUri);
if (_metadata?.Metadata.Subscription.RequiresAuthentication ?? false)
{
var token = await _tokenProvider.GetAccessTokenAsync(_options!, cancellationToken).ConfigureAwait(false);
if (token is not null)
{
var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType;
request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value);
}
}
return request;
}
private async Task QuarantineAsync(
RancherHubEventRecord record,
RancherHubEventBatch batch,
string reason,
VexConnectorContext context,
CancellationToken cancellationToken)
{
var metadata = BuildMetadata(builder => builder
.Add("rancher.event.id", record.Id)
.Add("rancher.event.type", record.Type)
.Add("rancher.event.channel", record.Channel)
.Add("rancher.event.quarantine", "true")
.Add("rancher.event.error", reason)
.Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor)
.Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false"));
var sourceUri = record.DocumentUri ?? _metadata?.Metadata.Subscription.EventsUri ?? _options!.DiscoveryUri;
var payload = Encoding.UTF8.GetBytes(record.RawJson);
var document = CreateRawDocument(VexDocumentFormat.Csaf, sourceUri, payload, metadata);
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Warning, "quarantine", "Rancher hub event moved to quarantine.", new Dictionary<string, object?>
{
["eventId"] = record.Id ?? "(missing)",
["reason"] = reason,
});
}
private static void AddQuarantineDigest(string key, HashSet<string> dedupeSet, List<string> digestHistory)
{
if (dedupeSet.Add(key))
{
digestHistory.Add(key);
}
}
private static string BuildQuarantineKey(RancherHubEventRecord record)
{
if (!string.IsNullOrWhiteSpace(record.Id))
{
return $"quarantine:{record.Id}";
}
Span<byte> hash = stackalloc byte[32];
var bytes = Encoding.UTF8.GetBytes(record.RawJson);
if (!SHA256.TryHashData(bytes, hash, out _))
{
using var sha = SHA256.Create();
hash = sha.ComputeHash(bytes);
}
return $"quarantine:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static string NormalizeDigest(string digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return digest;
}
var trimmed = digest.Trim();
return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
? trimmed.ToLowerInvariant()
: $"sha256:{trimmed.ToLowerInvariant()}";
}
private static VexDocumentFormat ResolveFormat(string? format)
{
if (string.IsNullOrWhiteSpace(format))
{
return VexDocumentFormat.Csaf;
}
return format.ToLowerInvariant() switch
{
"csaf" or "csaf_json" or "json" => VexDocumentFormat.Csaf,
"cyclonedx" or "cyclonedx_vex" => VexDocumentFormat.CycloneDx,
"openvex" => VexDocumentFormat.OpenVex,
"oci" or "oci_attestation" or "attestation" => VexDocumentFormat.OciAttestation,
_ => VexDocumentFormat.Csaf,
};
}
private sealed record EventProcessingResult(VexRawDocument? ProcessedDocument, bool Quarantined, DateTimeOffset? PublishedAt)
{
public static EventProcessingResult QuarantinedOnly { get; } = new(null, true, null);
public static EventProcessingResult Skipped { get; } = new(null, false, null);
}
}

View File

@@ -0,0 +1,98 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
public sealed record RancherHubCheckpointState(
string? Cursor,
DateTimeOffset? LastPublishedAt,
DateTimeOffset? EffectiveSince,
ImmutableArray<string> Digests);
public sealed class RancherHubCheckpointManager
{
private const string CheckpointPrefix = "checkpoint:";
private readonly IVexConnectorStateRepository _repository;
public RancherHubCheckpointManager(IVexConnectorStateRepository repository)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
}
public async ValueTask<RancherHubCheckpointState> LoadAsync(string connectorId, VexConnectorContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var state = await _repository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false);
var cursor = ExtractCursor(state?.DocumentDigests ?? ImmutableArray<string>.Empty);
var digests = ExtractDigests(state?.DocumentDigests ?? ImmutableArray<string>.Empty);
var lastPublishedAt = state?.LastUpdated;
var effectiveSince = context.Since;
if (context.Settings.Values.TryGetValue("checkpoint", out var checkpointOverride) && !string.IsNullOrWhiteSpace(checkpointOverride))
{
cursor = checkpointOverride;
digests = ImmutableArray<string>.Empty;
}
if (effectiveSince is null && lastPublishedAt is not null)
{
effectiveSince = lastPublishedAt;
}
if (effectiveSince is not null && lastPublishedAt is not null && effectiveSince < lastPublishedAt)
{
digests = ImmutableArray<string>.Empty;
}
return new RancherHubCheckpointState(cursor, lastPublishedAt, effectiveSince, digests);
}
public ValueTask SaveAsync(string connectorId, string? cursor, DateTimeOffset? lastPublishedAt, ImmutableArray<string> digests, CancellationToken cancellationToken)
{
var entries = ImmutableArray.CreateBuilder<string>();
if (!string.IsNullOrWhiteSpace(cursor))
{
entries.Add($"{CheckpointPrefix}{cursor}");
}
foreach (var digest in digests)
{
if (string.IsNullOrWhiteSpace(digest))
{
continue;
}
if (digest.StartsWith(CheckpointPrefix, StringComparison.Ordinal))
{
continue;
}
entries.Add(digest);
}
var state = new VexConnectorState(connectorId, lastPublishedAt, entries.ToImmutable());
return _repository.SaveAsync(state, cancellationToken);
}
private static string? ExtractCursor(ImmutableArray<string> digests)
{
foreach (var entry in digests)
{
if (entry.StartsWith(CheckpointPrefix, StringComparison.Ordinal))
{
return entry[CheckpointPrefix.Length..];
}
}
return null;
}
private static ImmutableArray<string> ExtractDigests(ImmutableArray<string> digests)
=> digests.Where(d => !d.StartsWith(CheckpointPrefix, StringComparison.Ordinal)).ToImmutableArray();
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-SUSE-01-001 Rancher hub discovery & auth|Team Excititor Connectors SUSE|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added Rancher hub options/token provider, discovery metadata loader with offline snapshots + caching, connector shell, DI wiring, and unit tests covering network/offline paths.|
|EXCITITOR-CONN-SUSE-01-002 Checkpointed event ingestion|Team Excititor Connectors SUSE|EXCITITOR-CONN-SUSE-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads.<br>2025-10-19: Prereqs EXCITITOR-CONN-SUSE-01-001 & EXCITITOR-STORAGE-01-003 confirmed complete; initiating checkpoint/resume implementation plan.|
|EXCITITOR-CONN-SUSE-01-003 Trust metadata & policy hints|Team Excititor Connectors SUSE|EXCITITOR-CONN-SUSE-01-002, EXCITITOR-POLICY-01-001|TODO Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine.|

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector for Ubuntu CSAF advisories (USN VEX data), managing discovery, incremental pulls, and raw document persistence.
## Scope
- Ubuntu CSAF metadata discovery, release channel awareness, and pagination handling.
- HTTP client with retries/backoff, checksum validation, and deduplication.
- Mapping Ubuntu identifiers (USN numbers, package metadata) into connector metadata for downstream policy.
- Emitting trust configuration (GPG fingerprints, cosign options) for policy weighting.
## Participants
- Worker schedules regular pulls; WebService can initiate manual ingest/resume.
- CSAF normalizer converts raw documents into claims.
- Policy engine leverages connector-supplied trust metadata.
## Interfaces & contracts
- Implements `IVexConnector`, using shared abstractions for HTTP/resume markers and telemetry.
- Provides options for release channels (stable/LTS) and offline seed bundles.
## In/Out of scope
In: data fetching, metadata mapping, raw persistence, trust hints.
Out: normalization/export, storage internals, attestation.
## Observability & security expectations
- Log release window fetch metrics, rate limits, and deduplication stats; mask secrets.
- Emit counters for newly ingested vs unchanged USNs and quota usage.
## Tests
- Connector tests with mocked Ubuntu CSAF endpoints will live in `../StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests`.

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
public sealed class UbuntuConnectorOptions
{
public const string HttpClientName = "excititor.connector.ubuntu.catalog";
/// <summary>
/// Root index that lists Ubuntu CSAF channels.
/// </summary>
public Uri IndexUri { get; set; } = new("https://ubuntu.com/security/csaf/index.json");
/// <summary>
/// Channels to include (e.g. stable, esm, lts).
/// </summary>
public IList<string> Channels { get; } = new List<string> { "stable" };
/// <summary>
/// Duration to cache discovery metadata.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(4);
/// <summary>
/// Prefer offline snapshot when available.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Optional file path for offline index snapshot.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// Controls persistence of network responses to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
public void Validate(IFileSystem? fileSystem = null)
{
if (IndexUri is null || !IndexUri.IsAbsoluteUri)
{
throw new InvalidOperationException("IndexUri must be an absolute URI.");
}
if (IndexUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("IndexUri must use HTTP or HTTPS.");
}
if (Channels.Count == 0)
{
throw new InvalidOperationException("At least one channel must be specified.");
}
for (var i = Channels.Count - 1; i >= 0; i--)
{
if (string.IsNullOrWhiteSpace(Channels[i]))
{
Channels.RemoveAt(i);
}
}
if (Channels.Count == 0)
{
throw new InvalidOperationException("Channel names cannot be empty.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("MetadataCacheDuration must be positive.");
}
if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
public sealed class UbuntuConnectorOptionsValidator : IVexConnectorOptionsValidator<UbuntuConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public UbuntuConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(VexConnectorDescriptor descriptor, UbuntuConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.DependencyInjection;
public static class UbuntuConnectorServiceCollectionExtensions
{
public static IServiceCollection AddUbuntuCsafConnector(this IServiceCollection services, Action<UbuntuConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<UbuntuConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddSingleton<IVexConnectorOptionsValidator<UbuntuConnectorOptions>, UbuntuConnectorOptionsValidator>();
services.AddHttpClient(UbuntuConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Ubuntu.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<UbuntuCatalogLoader>();
services.AddSingleton<IVexConnector, UbuntuCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,248 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
public sealed class UbuntuCatalogLoader
{
public const string CachePrefix = "StellaOps.Excititor.Connectors.Ubuntu.CSAF.Index";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<UbuntuCatalogLoader> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _semaphore = new(1, 1);
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
public UbuntuCatalogLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IFileSystem fileSystem,
ILogger<UbuntuCatalogLoader> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<UbuntuCatalogResult> LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
options.Validate(_fileSystem);
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
CacheEntry? entry = null;
if (options.PreferOfflineSnapshot)
{
entry = LoadFromOffline(options);
if (entry is null)
{
throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found.");
}
}
else
{
entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false)
?? LoadFromOffline(options);
}
if (entry is null)
{
throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot.");
}
var cacheOptions = new MemoryCacheEntryOptions();
if (entry.MetadataCacheDuration > TimeSpan.Zero)
{
cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration);
}
_memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions);
return entry.ToResult(fromCache: false);
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName);
using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var metadata = ParseMetadata(payload, options.Channels);
var now = _timeProvider.GetUtcNow();
var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false);
PersistSnapshotIfNeeded(options, metadata, now);
return entry;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri);
return null;
}
}
private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options)
{
if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(options.OfflineSnapshotPath))
{
_logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath);
var snapshot = JsonSerializer.Deserialize<UbuntuCatalogSnapshot>(payload, _serializerOptions);
if (snapshot is null)
{
throw new InvalidOperationException("Offline snapshot payload was empty.");
}
return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath);
return null;
}
}
private UbuntuCatalogMetadata ParseMetadata(string payload, IList<string> channels)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Ubuntu index payload was empty.");
}
using var document = JsonDocument.Parse(payload);
var root = document.RootElement;
var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated)
? generated
: _timeProvider.GetUtcNow();
var channelSet = new HashSet<string>(channels, StringComparer.OrdinalIgnoreCase);
if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array)
{
throw new InvalidOperationException("Ubuntu index did not include a channels array.");
}
var builder = ImmutableArray.CreateBuilder<UbuChannelCatalog>();
foreach (var channelElement in channelsElement.EnumerateArray())
{
var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name))
{
continue;
}
if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri))
{
_logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name);
continue;
}
string? sha256 = null;
if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String)
{
sha256 = shaElement.GetString();
}
DateTimeOffset? lastUpdated = null;
if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated))
{
lastUpdated = updated;
}
builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated));
}
if (builder.Count == 0)
{
throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index.");
}
return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable());
}
private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt)
{
if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return;
}
try
{
var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt);
var payload = JsonSerializer.Serialize(snapshot, _serializerOptions);
_fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath);
}
}
private static string CreateCacheKey(UbuntuConnectorOptions options)
=> $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}";
private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot)
{
public bool IsExpired(DateTimeOffset now)
=> MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration;
public UbuntuCatalogResult ToResult(bool fromCache)
=> new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot);
}
private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt);
}
public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray<UbuChannelCatalog> Channels);
public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated);
public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot);

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,8 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-UBUNTU-01-001 Ubuntu CSAF discovery & channels|Team Excititor Connectors Ubuntu|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added Ubuntu connector project with configurable channel options, catalog loader (network/offline), DI wiring, and discovery unit tests.|
|EXCITITOR-CONN-UBUNTU-01-002 Incremental fetch & deduplication|Team Excititor Connectors Ubuntu|EXCITITOR-CONN-UBUNTU-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.|
|EXCITITOR-CONN-UBUNTU-01-003 Trust metadata & provenance|Team Excititor Connectors Ubuntu|EXCITITOR-CONN-UBUNTU-01-002, EXCITITOR-POLICY-01-001|TODO Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics.|
> Remark (2025-10-19, EXCITITOR-CONN-UBUNTU-01-002): Prerequisites EXCITITOR-CONN-UBUNTU-01-001 and EXCITITOR-STORAGE-01-003 verified as **DONE**; advancing to DOING per Wave 0 kickoff.

Some files were not shown because too many files have changed in this diff Show More