consolidation of some of the modules, localization fixes, product advisories work, qa work

This commit is contained in:
master
2026-03-05 03:54:22 +02:00
parent 7bafcc3eef
commit 8e1cb9448d
3878 changed files with 72600 additions and 46861 deletions

View File

@@ -27,7 +27,7 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.ProofService\StellaOps.Concelier.ProofService.csproj" />
<ProjectReference Include="..\..\..\Feedser\StellaOps.Feedser.BinaryAnalysis\StellaOps.Feedser.BinaryAnalysis.csproj" />
<ProjectReference Include="..\..\StellaOps.Feedser.BinaryAnalysis\StellaOps.Feedser.BinaryAnalysis.csproj" />
</ItemGroup>
</Project>

View File

@@ -13,8 +13,8 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.SourceIntel\StellaOps.Concelier.SourceIntel.csproj" />
<ProjectReference Include="..\..\..\Feedser\StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj" />
<ProjectReference Include="..\..\..\Feedser\StellaOps.Feedser.BinaryAnalysis\StellaOps.Feedser.BinaryAnalysis.csproj" />
<ProjectReference Include="..\..\StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj" />
<ProjectReference Include="..\..\StellaOps.Feedser.BinaryAnalysis\StellaOps.Feedser.BinaryAnalysis.csproj" />
<ProjectReference Include="..\..\..\Attestor\__Libraries\StellaOps.Attestor.ProofChain\StellaOps.Attestor.ProofChain.csproj" />
</ItemGroup>

View File

@@ -0,0 +1,24 @@
# Excititor S3 Artifact Store Agent Charter
## Mission
Maintain the S3-backed artifact store client and DI wiring for Excititor exports.
## Responsibilities
- Implement S3 client interactions for artifact storage.
- Provide dependency injection wiring and options handling.
- Keep storage operations deterministic and failure-aware.
## Required Reading
- docs/modules/excititor/architecture.md
- docs/modules/platform/architecture-overview.md
## Definition of Done
- S3 client operations are validated and tested.
- Error handling is predictable and logged.
## Working Agreement
- 1. Update task status to DOING/DONE in the sprint file and local TASKS.md.
- 2. Review this charter and required docs before coding.
- 3. Keep outputs deterministic (ordering, timestamps, hashes) and offline-friendly.
- 4. Add tests for error paths and option validation.
- 5. Revert to TODO if paused; capture context in PR notes.

View File

@@ -0,0 +1,38 @@
using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Export;
namespace StellaOps.Excititor.ArtifactStores.S3.Extensions;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddVexS3ArtifactClient(this IServiceCollection services, Action<S3ArtifactClientOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddSingleton(CreateS3Client);
services.AddSingleton<IS3ArtifactClient, S3ArtifactClient>();
return services;
}
private static IAmazonS3 CreateS3Client(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<S3ArtifactClientOptions>>().Value;
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
ForcePathStyle = options.ForcePathStyle,
};
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
{
config.ServiceURL = options.ServiceUrl;
}
return new AmazonS3Client(config);
}
}

View File

@@ -0,0 +1,85 @@
using Amazon.S3;
using Amazon.S3.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Export;
namespace StellaOps.Excititor.ArtifactStores.S3;
public sealed class S3ArtifactClientOptions
{
public string Region { get; set; } = "us-east-1";
public string? ServiceUrl { get; set; }
= null;
public bool ForcePathStyle { get; set; }
= true;
}
public sealed class S3ArtifactClient : IS3ArtifactClient
{
private readonly IAmazonS3 _s3;
private readonly ILogger<S3ArtifactClient> _logger;
public S3ArtifactClient(IAmazonS3 s3, ILogger<S3ArtifactClient> logger)
{
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<bool> ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken)
{
try
{
var metadata = await _s3.GetObjectMetadataAsync(bucketName, key, cancellationToken).ConfigureAwait(false);
return metadata.HttpStatusCode == System.Net.HttpStatusCode.OK;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return false;
}
}
public async Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary<string, string> metadata, CancellationToken cancellationToken)
{
var request = new PutObjectRequest
{
BucketName = bucketName,
Key = key,
InputStream = content,
AutoCloseStream = false,
};
foreach (var kvp in metadata)
{
request.Metadata[kvp.Key] = kvp.Value;
}
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Uploaded object {Bucket}/{Key}", bucketName, key);
}
public async Task<Stream?> GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken)
{
try
{
var response = await _s3.GetObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false);
var buffer = new MemoryStream();
await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
return buffer;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Object {Bucket}/{Key} not found", bucketName, key);
return null;
}
}
public async Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken)
{
await _s3.DeleteObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Deleted object {Bucket}/{Key}", bucketName, key);
}
}

View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Options" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Export\StellaOps.Excititor.Export.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,10 @@
# Excititor S3 Artifact Store Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0293-M | DONE | Revalidated 2026-01-07; open findings tracked in audit report. |
| AUDIT-0293-T | DONE | Revalidated 2026-01-07; open findings tracked in audit report. |
| AUDIT-0293-A | TODO | Revalidated 2026-01-07 (open findings). |

View File

@@ -0,0 +1,34 @@
# AGENTS
## Role
Builds and verifies in-toto/DSSE attestations for Excititor exports and integrates with Rekor v2 transparency logs.
## Scope
- Attestation envelope builders, signing workflows (keyless/keyed), and predicate model definitions.
- Rekor v2 client implementation (submit, verify, poll inclusion) with retry/backoff policies.
- Verification utilities reused by Worker for periodic revalidation.
- Configuration bindings for signer identity, Rekor endpoints, and offline bundle operation.
## Participants
- Export module calls into this layer to generate attestations after export artifacts are produced.
- WebService and Worker consume verification helpers to ensure stored envelopes remain valid.
- CLI `excititor verify` leverages verification services through WebService endpoints.
## Interfaces & contracts
- `IExportAttestor`, `ITransparencyLogClient`, predicate DTOs, and verification result records.
- Extension methods to register attestation services in DI across WebService/Worker.
## In/Out of scope
In: attestation creation, verification, Rekor integration, signer configuration.
Out: export artifact generation, storage persistence, CLI interaction layers.
## Observability & security expectations
- Structured logs for signing/verification with envelope digest, Rekor URI, and latency; never log private keys.
- Metrics for attestation successes/failures and Rekor submission durations.
## Tests
- Unit tests and integration stubs (with fake Rekor) will live in `../StellaOps.Excititor.Attestation.Tests`.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,9 @@
// Re-export DSSE types from Core for backwards compatibility
// The canonical types are in StellaOps.Excititor.Core.Dsse
global using DsseEnvelope = StellaOps.Excititor.Core.Dsse.DsseEnvelope;
global using DsseSignature = StellaOps.Excititor.Core.Dsse.DsseSignature;
namespace StellaOps.Excititor.Attestation.Dsse;
// Types re-exported from StellaOps.Excititor.Core.Dsse via global using

View File

@@ -0,0 +1,84 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Core;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Dsse;
public sealed class VexDsseBuilder
{
internal const string PayloadType = "application/vnd.in-toto+json";
private readonly IVexSigner _signer;
private readonly ILogger<VexDsseBuilder> _logger;
private readonly JsonSerializerOptions _serializerOptions;
public VexDsseBuilder(IVexSigner signer, ILogger<VexDsseBuilder> logger)
{
_signer = signer ?? throw new ArgumentNullException(nameof(signer));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_serializerOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
WriteIndented = false,
};
_serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase));
}
public async ValueTask<DsseEnvelope> CreateEnvelopeAsync(
VexAttestationRequest request,
IReadOnlyDictionary<string, string>? metadata,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var predicate = VexAttestationPredicate.FromRequest(request, metadata);
var subject = new VexInTotoSubject(
request.ExportId,
new Dictionary<string, string>(StringComparer.Ordinal)
{
{ request.Artifact.Algorithm.ToLowerInvariant(), request.Artifact.Digest }
});
var statement = new VexInTotoStatement(
VexInTotoStatement.InTotoType,
"https://stella-ops.org/attestations/vex-export",
new[] { subject },
predicate);
var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(statement, _serializerOptions);
var signatureResult = await _signer.SignAsync(payloadBytes, cancellationToken).ConfigureAwait(false);
var envelope = new DsseEnvelope(
Convert.ToBase64String(payloadBytes),
PayloadType,
new[] { new DsseSignature(signatureResult.Signature, signatureResult.KeyId) });
_logger.LogDebug("DSSE envelope created for export {ExportId}", request.ExportId);
return envelope;
}
public static string ComputeEnvelopeDigest(DsseEnvelope envelope)
{
ArgumentNullException.ThrowIfNull(envelope);
var envelopeJson = JsonSerializer.Serialize(envelope, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
});
var bytes = Encoding.UTF8.GetBytes(envelopeJson);
var hash = SHA256.HashData(bytes);
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
}

View File

@@ -0,0 +1,159 @@
# EXCITITOR-ATTEST-01-003 - Verification & Observability Plan
- **Date:** 2025-10-19
- **Status:** In progress (2025-10-22)
- **Owner:** Team Excititor Attestation
- **Related tasks:** EXCITITOR-ATTEST-01-003 (Wave 0), EXCITITOR-WEB-01-003/004, EXCITITOR-WORKER-01-003
- **Prerequisites satisfied:** EXCITITOR-ATTEST-01-002 (Rekor v2 client integration)
## 1. Objectives
1. Provide deterministic attestation verification helpers consumable by Excititor WebService (`/excititor/verify`, `/excititor/export*`) and Worker re-verification loops.
2. Surface structured diagnostics for success, soft failures, and hard failures (signature mismatch, Rekor gaps, artifact digest drift).
3. Emit observability signals (logs, metrics, optional tracing) that can run offline and degrade gracefully when transparency services are unreachable.
4. Add regression tests (unit + integration) covering positive path, negative path, and offline fallback scenarios.
## 2. Deliverables
- `IVexAttestationVerifier` abstraction + `VexAttestationVerifier` implementation inside `StellaOps.Excititor.Attestation`, encapsulating DSSE validation, predicate checks, artifact digest confirmation, Rekor inclusion verification, and deterministic diagnostics.
- DI wiring (extension method) for registering verifier + instrumentation dependencies alongside the existing signer/rekor client.
- Shared `VexAttestationDiagnostics` record describing normalized diagnostic keys consumed by Worker/WebService logging.
- Metrics utility (`AttestationMetrics`) exposing counters/histograms via `System.Diagnostics.Metrics`, exported under `StellaOps.Excititor.Attestation` meter.
- Activity source (`AttestationActivitySource`) for optional tracing spans around sign/verify operations.
- 2025-11-05: Implemented `VexAttestationDiagnostics`, activity tagging via `VexAttestationActivitySource`, and updated verifier/tests to emit structured failure reasons.
- 2025-11-05 (pm): Worker attestation verifier now records structured diagnostics/metrics and logs result/failure reasons using `VexAttestationDiagnostics`; attestation success/failure labels propagate to verification counters.
- Documentation updates (`EXCITITOR-ATTEST-01-003-plan.md`, `TASKS.md` notes) describing instrumentation + test expectations.
- Test coverage in `StellaOps.Excititor.Attestation.Tests` (unit) and scaffolding notes for WebService/Worker integration tests.
## 3. Verification Flow
### 3.1 Inputs
- `VexAttestationRequest` from Core (contains export identifiers, artifact digest, metadata, source providers).
- Optional Rekor reference from previous signing (`VexAttestationMetadata.Rekor`).
- Configured policies (tolerated clock skew, Rekor verification toggle, offline mode flag, maximum metadata drift).
### 3.2 Steps
1. **Envelope decode** - retrieve DSSE envelope + predicate from storage (Worker) or request payload (WebService), canonicalize JSON, compute digest, compare with metadata `envelopeDigest`.
2. **Subject validation** - ensure subject digest matches exported artifact digest (algorithm & value) and export identifier matches `request.ExportId`.
3. **Signature verification** - delegate to signer/verifier abstraction (cosign/x509) using configured trust anchors; record `signature_state` diagnostic (verified, skipped_offline, failed).
4. **Provenance checks** - confirm predicate type (`https://stella-ops.org/attestations/vex-export`) and metadata shape; enforce deterministic timestamp tolerance.
5. **Transparency log** - if Rekor reference present and verification enabled, call `ITransparencyLogClient.VerifyAsync` with retry/backoff budget; support offline bypass with diagnostic `rekor_state=unreachable`.
6. **Result aggregation** - produce `VexAttestationVerification` containing `IsValid` flag and diagnostics map (includes `failure_reason` when invalid).
### 3.3 Failure Categories & Handling
| Category | Detection | Handling |
|---|---|---|
| Signature mismatch | Signer verification failure or subject digest mismatch | Mark invalid, emit warning log, increment `verify.failed` counter with `reason=signature_mismatch`. |
| Rekor absence/stale | Rekor verify returns false | Mark invalid unless offline mode configured; log with correlation ID; `reason=rekor_missing`. |
| Predicate schema drift | Predicate type or required fields missing | Mark invalid, include `reason=predicate_invalid`. |
| Time skew | `signedAt` older than policy threshold | Mark invalid (hard) or warn (soft) per options; include `reason=stale_attestation`. |
| Unexpected metadata | Unknown export format, provider mismatch | Mark invalid; `reason=metadata_mismatch`. |
| Offline Rekor | HTTP client throws | Mark soft failure if `AllowOfflineTransparency` true; degrade metrics with `rekor_state=offline`. |
## 4. Observability
### 4.1 Metrics (Meter name: `StellaOps.Excititor.Attestation`)
| Metric | Type | Dimensions | Description |
|---|---|---|---|
| `stellaops.excititor.attestation.verify.total` | Counter<long> | `result` (`success`/`failure`/`soft_failure`), `component` (`webservice`/`worker`), `reverify` (`true`/`false`) | Counts verification attempts. |
| `stellaops.excititor.attestation.verify.duration.ms` | Histogram<double> | `component`, `result` | Measures end-to-end verification latency. |
| `stellaops.excititor.attestation.verify.rekor.calls` | Counter<long> | `result` (`verified`/`unreachable`/`skipped`) | Rekor verification outcomes. |
| `stellaops.excititor.attestation.verify.cache.hit` | Counter<long> | `hit` (`true`/`false`) | Tracks reuse of cached verification results (Worker loop). |
Metrics must register via static helper using `Meter` and support offline operation (no exporter dependency). Histogram records double milliseconds; use `Stopwatch.GetElapsedTime` for monotonic timing.
### 4.2 Logging
- Use structured logs (`ILogger<VexAttestationVerifier>`) with event IDs: `AttestationVerified` (Information), `AttestationVerificationFailed` (Warning), `AttestationVerificationError` (Error).
- Include correlation ID (`request.QuerySignature.Value`), `exportId`, `envelopeDigest`, `rekorLocation`, `reason`, and `durationMs`.
- Avoid logging private keys or full envelope; log envelope digest only. For debug builds, gate optional envelope JSON behind `LogLevel.Trace` and configuration flag.
### 4.3 Tracing
- Activity source name `StellaOps.Excititor.Attestation` with spans `attestation.verify` (parent from WebService request or Worker job) including tags: `stellaops.export_id`, `stellaops.result`, `stellaops.rekor.state`.
- Propagate Activity through Rekor client via `HttpClient` instrumentation (auto instrumentation available).
## 5. Integration Points
### 5.1 WebService
- Inject `IVexAttestationVerifier` into export endpoints and `/excititor/verify` handler.
- Persist verification result diagnostics alongside response payload for deterministic clients.
- Return HTTP 200 with `{ valid: true }` when verified; 409 for invalid attestation with diagnostics JSON; 503 when Rekor unreachable and offline override disabled.
- Add caching for idempotent verification (e.g., by envelope digest) to reduce Rekor calls and surface via metrics.
### 5.2 Worker
- Schedule background job (`EXCITITOR-WORKER-01-003`) to re-verify stored attestations on TTL (default 12h) using new verifier; on failure, flag export for re-sign and notify via event bus (future task).
- Emit logs/metrics with `component=worker`; include job IDs and next scheduled run.
- Provide cancellation-aware loops (respect `CancellationToken`) and deterministic order (sorted by export id).
### 5.3 Storage / Cache Hooks
- Store latest verification status and diagnostics in attestation metadata collection (Mongo) keyed by `envelopeDigest` + `artifactDigest` to avoid duplicate work.
- Expose read API (via WebService) for clients to fetch last verification timestamp + result.
## 6. Test Strategy
### 6.1 Unit Tests (`StellaOps.Excititor.Attestation.Tests`)
- `VexAttestationVerifierTests.VerifyAsync_Succeeds_WhenSignatureAndRekorValid` - uses fake signer/verifier + in-memory Rekor client returning success.
- `...ReturnsSoftFailure_WhenRekorOfflineAndAllowed` - ensure `IsValid=true`, diagnostic `rekor_state=offline`, metric increments `result=soft_failure`.
- `...Fails_WhenDigestMismatch` - ensures invalid result, log entry recorded, metrics increment `result=failure` with `reason=signature_mismatch`.
- `...Fails_WhenPredicateTypeUnexpected` - invalid with `reason=predicate_invalid`.
- `...RespectsCancellation` - cancellation token triggered before Rekor call results in `OperationCanceledException` and no metrics increments beyond started attempt.
### 6.2 WebService Integration Tests (`StellaOps.Excititor.WebService.Tests`)
- `VerifyEndpoint_Returns200_OnValidAttestation` - mocks verifier to return success, asserts response payload, metrics stub invoked.
- `VerifyEndpoint_Returns409_OnInvalid` - invalid diag forwarded, ensures logging occurs.
- `ExportEndpoint_IncludesVerificationDiagnostics` - ensures signed export responses include last verification metadata.
### 6.3 Worker Tests (`StellaOps.Excititor.Worker.Tests`)
- `ReverificationJob_RequeuesOnFailure` - invalid result triggers requeue/backoff.
- `ReverificationJob_PersistsStatusAndMetrics` - success path updates repository & metrics.
### 6.4 Determinism/Regression
- Golden test verifying that identical inputs produce identical diagnostics dictionaries (sorted keys).
- Ensure metrics dimensions remain stable via snapshot test (e.g., capturing tags in fake meter listener).
## 7. Implementation Sequencing
1. Introduce verifier abstraction + implementation with basic tests (signature + Rekor success/failure).
2. Add observability helpers (metrics, activity, logging) and wire into verifier; extend tests to assert instrumentation (using in-memory listener/log sink).
3. Update WebService DI/service layer to use verifier; craft endpoint integration tests.
4. Update Worker scheduling code to call verifier & emit metrics.
5. Wire persistence/caching and document configuration knobs (retry, offline, TTL).
6. Finalize documentation (architecture updates, runbook entries) before closing task.
## 8. Configuration Defaults
- `AttestationVerificationOptions` (new): `RequireRekor=true`, `AllowOfflineTransparency=false`, `MaxClockSkew=PT5M`, `ReverifyInterval=PT12H`, `CacheWindow=PT1H`.
- Options bind from configuration section `Excititor:Attestation` across WebService/Worker; offline kit ships defaults.
## 9. Open Questions
- Should verification gracefully accept legacy predicate types (pre-1.0) or hard fail? (Proposed: allow via allowlist with warning diagnostics.)
- Do we need cross-module eventing when verification fails (e.g., notify Export module) or is logging sufficient in Wave 0? (Proposed: log + metrics now, escalate in later wave.)
- Confirm whether Worker re-verification writes to Mongo or triggers Export module to re-sign artifacts automatically; placeholder: record status + timestamp only.
## 10. Acceptance Criteria
- Plan approved by Attestation + WebService + Worker leads.
- Metrics/logging names peer-reviewed to avoid collisions.
- Test backlog items entered into respective `TASKS.md` once implementation starts.
- Documentation (this plan) linked from `TASKS.md` notes for discoverability.
## 11. 2025-10-22 Progress Notes
- Implemented `IVexAttestationVerifier`/`VexAttestationVerifier` with structural validation (subject/predicate checks, digest comparison, Rekor probes) and diagnostics map.
- Added `VexAttestationVerificationOptions` (RequireTransparencyLog, AllowOfflineTransparency, MaxClockSkew) and wired configuration through WebService DI.
- Created `VexAttestationMetrics` (`excititor.attestation.verify_total`, `excititor.attestation.verify_duration_seconds`) and hooked into verification flow with component/rekor tags.
- `VexAttestationClient.VerifyAsync` now delegates to the verifier; DI registers metrics + verifier via `AddVexAttestation`.
- Added unit coverage in `VexAttestationVerifierTests` (happy path, digest mismatch, offline Rekor) and updated client/export/webservice stubs to new verification signature.

View File

@@ -0,0 +1,249 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Core.Evidence;
using System;
using System.Collections.Immutable;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Evidence;
/// <summary>
/// Default implementation of <see cref="IVexEvidenceAttestor"/> that creates DSSE attestations for evidence manifests.
/// </summary>
public sealed class VexEvidenceAttestor : IVexEvidenceAttestor
{
internal const string PayloadType = "application/vnd.in-toto+json";
private readonly IVexSigner _signer;
private readonly TimeProvider _timeProvider;
private readonly ILogger<VexEvidenceAttestor> _logger;
private readonly JsonSerializerOptions _serializerOptions;
public VexEvidenceAttestor(
IVexSigner signer,
ILogger<VexEvidenceAttestor> logger,
TimeProvider? timeProvider = null)
{
_signer = signer ?? throw new ArgumentNullException(nameof(signer));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_serializerOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
_serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase));
}
public async ValueTask<VexEvidenceAttestationResult> AttestManifestAsync(
VexLockerManifest manifest,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(manifest);
var attestedAt = _timeProvider.GetUtcNow();
var attestationId = CreateAttestationId(manifest, attestedAt);
// Build in-toto statement
var predicate = VexEvidenceAttestationPredicate.FromManifest(manifest);
var subject = new VexEvidenceInTotoSubject(
manifest.ManifestId,
ImmutableDictionary<string, string>.Empty.Add("sha256", manifest.MerkleRoot.Replace("sha256:", "")));
var statement = new InTotoStatementDto
{
Type = VexEvidenceInTotoStatement.InTotoStatementType,
PredicateType = VexEvidenceInTotoStatement.EvidenceLockerPredicateType,
Subject = new[] { new InTotoSubjectDto { Name = subject.Name, Digest = subject.Digest } },
Predicate = new InTotoPredicateDto
{
ManifestId = predicate.ManifestId,
Tenant = predicate.Tenant,
MerkleRoot = predicate.MerkleRoot,
ItemCount = predicate.ItemCount,
CreatedAt = predicate.CreatedAt,
Metadata = predicate.Metadata.Count > 0 ? predicate.Metadata : null
}
};
// Serialize and sign
var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(statement, _serializerOptions);
var signatureResult = await _signer.SignAsync(payloadBytes, cancellationToken).ConfigureAwait(false);
// Build DSSE envelope
var envelope = new DsseEnvelope(
Convert.ToBase64String(payloadBytes),
PayloadType,
new[] { new DsseSignature(signatureResult.Signature, signatureResult.KeyId) });
var envelopeJson = JsonSerializer.Serialize(envelope, _serializerOptions);
var envelopeHash = ComputeHash(envelopeJson);
// Create signed manifest
var signedManifest = manifest.WithSignature(signatureResult.Signature);
_logger.LogDebug(
"Evidence attestation created for manifest {ManifestId}: attestation={AttestationId} hash={Hash}",
manifest.ManifestId,
attestationId,
envelopeHash);
return new VexEvidenceAttestationResult(
signedManifest,
envelopeJson,
envelopeHash,
attestationId,
attestedAt);
}
public ValueTask<VexEvidenceVerificationResult> VerifyAttestationAsync(
VexLockerManifest manifest,
string dsseEnvelopeJson,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(manifest);
if (string.IsNullOrWhiteSpace(dsseEnvelopeJson))
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure("DSSE envelope is required."));
}
try
{
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(dsseEnvelopeJson, _serializerOptions);
if (envelope is null)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure("Invalid DSSE envelope format."));
}
// Decode payload and verify it matches the manifest
var payloadBytes = Convert.FromBase64String(envelope.Payload);
var statement = JsonSerializer.Deserialize<InTotoStatementDto>(payloadBytes, _serializerOptions);
if (statement is null)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure("Invalid in-toto statement format."));
}
// Verify statement type
if (statement.Type != VexEvidenceInTotoStatement.InTotoStatementType)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
$"Invalid statement type: expected {VexEvidenceInTotoStatement.InTotoStatementType}, got {statement.Type}"));
}
// Verify predicate type
if (statement.PredicateType != VexEvidenceInTotoStatement.EvidenceLockerPredicateType)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
$"Invalid predicate type: expected {VexEvidenceInTotoStatement.EvidenceLockerPredicateType}, got {statement.PredicateType}"));
}
// Verify manifest ID matches
if (statement.Predicate?.ManifestId != manifest.ManifestId)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
$"Manifest ID mismatch: expected {manifest.ManifestId}, got {statement.Predicate?.ManifestId}"));
}
// Verify Merkle root matches
if (statement.Predicate?.MerkleRoot != manifest.MerkleRoot)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
$"Merkle root mismatch: expected {manifest.MerkleRoot}, got {statement.Predicate?.MerkleRoot}"));
}
// Verify item count matches
if (statement.Predicate?.ItemCount != manifest.Items.Length)
{
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
$"Item count mismatch: expected {manifest.Items.Length}, got {statement.Predicate?.ItemCount}"));
}
var diagnostics = ImmutableDictionary.CreateBuilder<string, string>();
diagnostics.Add("envelope_hash", ComputeHash(dsseEnvelopeJson));
diagnostics.Add("verified_at", _timeProvider.GetUtcNow().ToString("O", CultureInfo.InvariantCulture));
_logger.LogDebug("Evidence attestation verified for manifest {ManifestId}", manifest.ManifestId);
return ValueTask.FromResult(VexEvidenceVerificationResult.Success(diagnostics.ToImmutable()));
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to parse DSSE envelope for manifest {ManifestId}", manifest.ManifestId);
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure($"JSON parse error: {ex.Message}"));
}
catch (FormatException ex)
{
_logger.LogWarning(ex, "Failed to decode base64 payload for manifest {ManifestId}", manifest.ManifestId);
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure($"Base64 decode error: {ex.Message}"));
}
}
private static string CreateAttestationId(VexLockerManifest manifest, DateTimeOffset timestamp)
{
var normalized = manifest.Tenant.ToLowerInvariant();
var date = timestamp.ToString("yyyyMMddHHmmssfff");
return $"attest:evidence:{normalized}:{date}";
}
private static string ComputeHash(string content)
{
var bytes = Encoding.UTF8.GetBytes(content);
var hash = SHA256.HashData(bytes);
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
// DTOs for JSON serialization
private sealed record InTotoStatementDto
{
[JsonPropertyName("_type")]
public string? Type { get; init; }
[JsonPropertyName("predicateType")]
public string? PredicateType { get; init; }
[JsonPropertyName("subject")]
public InTotoSubjectDto[]? Subject { get; init; }
[JsonPropertyName("predicate")]
public InTotoPredicateDto? Predicate { get; init; }
}
private sealed record InTotoSubjectDto
{
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("digest")]
public ImmutableDictionary<string, string>? Digest { get; init; }
}
private sealed record InTotoPredicateDto
{
[JsonPropertyName("manifestId")]
public string? ManifestId { get; init; }
[JsonPropertyName("tenant")]
public string? Tenant { get; init; }
[JsonPropertyName("merkleRoot")]
public string? MerkleRoot { get; init; }
[JsonPropertyName("itemCount")]
public int? ItemCount { get; init; }
[JsonPropertyName("createdAt")]
public DateTimeOffset? CreatedAt { get; init; }
[JsonPropertyName("metadata")]
public ImmutableDictionary<string, string>? Metadata { get; init; }
}
}

View File

@@ -0,0 +1,30 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Evidence;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Evidence;
namespace StellaOps.Excititor.Attestation.Extensions;
public static class VexAttestationServiceCollectionExtensions
{
public static IServiceCollection AddVexAttestation(this IServiceCollection services)
{
services.AddSingleton<VexDsseBuilder>();
services.AddSingleton<VexAttestationMetrics>();
services.AddSingleton<IVexAttestationVerifier, VexAttestationVerifier>();
services.AddSingleton<IVexAttestationClient, VexAttestationClient>();
services.AddSingleton<IVexEvidenceAttestor, VexEvidenceAttestor>();
return services;
}
public static IServiceCollection AddVexRekorClient(this IServiceCollection services, Action<RekorHttpClientOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddHttpClient<ITransparencyLogClient, RekorHttpClient>();
return services;
}
}

View File

@@ -0,0 +1,45 @@
using StellaOps.Excititor.Core;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.Attestation.Models;
public sealed record VexAttestationPredicate(
string ExportId,
string QuerySignature,
string ArtifactAlgorithm,
string ArtifactDigest,
VexExportFormat Format,
DateTimeOffset CreatedAt,
IReadOnlyList<string> SourceProviders,
IReadOnlyDictionary<string, string> Metadata)
{
public static VexAttestationPredicate FromRequest(
VexAttestationRequest request,
IReadOnlyDictionary<string, string>? metadata = null)
=> new(
request.ExportId,
request.QuerySignature.Value,
request.Artifact.Algorithm,
request.Artifact.Digest,
request.Format,
request.CreatedAt,
request.SourceProviders,
metadata is null ? ImmutableDictionary<string, string>.Empty : metadata.ToImmutableDictionary(StringComparer.Ordinal));
}
public sealed record VexInTotoSubject(
string Name,
IReadOnlyDictionary<string, string> Digest);
public sealed record VexInTotoStatement(
[property: JsonPropertyName("_type")] string Type,
string PredicateType,
IReadOnlyList<VexInTotoSubject> Subject,
VexAttestationPredicate Predicate)
{
public static readonly string InTotoType = "https://in-toto.io/Statement/v0.1";
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Attestation.Tests")]

View File

@@ -0,0 +1,12 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Signing;
public sealed record VexSignedPayload(string Signature, string? KeyId);
public interface IVexSigner
{
ValueTask<VexSignedPayload> SignAsync(ReadOnlyMemory<byte> payload, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="Microsoft.Extensions.Http" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Cryptography\StellaOps.Cryptography.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,6 @@
# Completed Tasks
|EXCITITOR-ATTEST-01-001 In-toto predicate & DSSE builder|Team Excititor Attestation|EXCITITOR-CORE-01-001|**DONE (2025-10-16)** Added deterministic in-toto predicate/statement models, DSSE envelope builder wired to signer abstraction, and attestation client producing metadata + diagnostics.|
|EXCITITOR-ATTEST-01-002 Rekor v2 client integration|Team Excititor Attestation|EXCITITOR-ATTEST-01-001|**DONE (2025-10-16)** Implemented Rekor HTTP client with retry/backoff, transparency log abstraction, DI helpers, and attestation client integration capturing Rekor metadata + diagnostics.|

View File

@@ -0,0 +1,10 @@
# Excititor Attestation Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0295-M | DONE | Revalidated 2026-01-07; open findings tracked in audit report. |
| AUDIT-0295-T | DONE | Revalidated 2026-01-07; open findings tracked in audit report. |
| AUDIT-0295-A | TODO | Revalidated 2026-01-07 (open findings). |

View File

@@ -0,0 +1,15 @@
using StellaOps.Excititor.Attestation.Dsse;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Transparency;
public sealed record TransparencyLogEntry(string Id, string Location, string? LogIndex, string? InclusionProofUrl);
public interface ITransparencyLogClient
{
ValueTask<TransparencyLogEntry> SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken);
ValueTask<bool> VerifyAsync(string entryLocation, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,92 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Attestation.Dsse;
using System.Net.Http.Json;
using System.Text.Json;
namespace StellaOps.Excititor.Attestation.Transparency;
internal sealed class RekorHttpClient : ITransparencyLogClient
{
private readonly HttpClient _httpClient;
private readonly RekorHttpClientOptions _options;
private readonly ILogger<RekorHttpClient> _logger;
public RekorHttpClient(HttpClient httpClient, IOptions<RekorHttpClientOptions> options, ILogger<RekorHttpClient> logger)
{
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (!string.IsNullOrWhiteSpace(_options.BaseAddress))
{
_httpClient.BaseAddress = new Uri(_options.BaseAddress, UriKind.Absolute);
}
if (!string.IsNullOrWhiteSpace(_options.ApiKey))
{
_httpClient.DefaultRequestHeaders.Add("Authorization", _options.ApiKey);
}
}
public async ValueTask<TransparencyLogEntry> SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(envelope);
var payload = JsonSerializer.Serialize(envelope);
using var content = new StringContent(payload);
content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
HttpResponseMessage? response = null;
for (var attempt = 0; attempt < _options.RetryCount; attempt++)
{
response = await _httpClient.PostAsync("/api/v2/log/entries", content, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
break;
}
_logger.LogWarning("Rekor submission failed with status {Status}; attempt {Attempt}", response.StatusCode, attempt + 1);
if (attempt + 1 < _options.RetryCount)
{
await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false);
}
}
if (response is null || !response.IsSuccessStatusCode)
{
throw new HttpRequestException($"Failed to submit attestation to Rekor ({response?.StatusCode}).");
}
var entryLocation = response.Headers.Location?.ToString() ?? string.Empty;
var body = await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken: cancellationToken).ConfigureAwait(false);
var entry = ParseEntryLocation(entryLocation, body);
_logger.LogInformation("Rekor entry recorded at {Location}", entry.Location);
return entry;
}
public async ValueTask<bool> VerifyAsync(string entryLocation, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(entryLocation))
{
return false;
}
var response = await _httpClient.GetAsync(entryLocation, cancellationToken).ConfigureAwait(false);
return response.IsSuccessStatusCode;
}
private static TransparencyLogEntry ParseEntryLocation(string location, JsonElement body)
{
var id = body.TryGetProperty("uuid", out var uuid) ? uuid.GetString() ?? string.Empty : Guid.NewGuid().ToString();
var logIndex = body.TryGetProperty("logIndex", out var logIndexElement) ? logIndexElement.GetString() : null;
string? inclusionProof = null;
if (body.TryGetProperty("verification", out var verification) && verification.TryGetProperty("inclusionProof", out var inclusion))
{
inclusionProof = inclusion.GetProperty("logIndex").GetRawText();
}
return new TransparencyLogEntry(id, location, logIndex, inclusionProof);
}
}

View File

@@ -0,0 +1,13 @@
namespace StellaOps.Excititor.Attestation.Transparency;
public sealed class RekorHttpClientOptions
{
public string BaseAddress { get; set; } = "https://rekor.sigstore.dev";
public string? ApiKey { get; set; }
= null;
public int RetryCount { get; set; } = 3;
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(2);
}

View File

@@ -0,0 +1,11 @@
using StellaOps.Excititor.Core;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Verification;
public interface IVexAttestationVerifier
{
ValueTask<VexAttestationVerification> VerifyAsync(VexAttestationVerificationRequest request, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,10 @@
using System.Diagnostics;
namespace StellaOps.Excititor.Attestation.Verification;
public static class VexAttestationActivitySource
{
public const string Name = "StellaOps.Excititor.Attestation";
public static readonly ActivitySource Value = new(Name);
}

View File

@@ -0,0 +1,35 @@
using System;
using System.Diagnostics.Metrics;
namespace StellaOps.Excititor.Attestation.Verification;
public sealed class VexAttestationMetrics : IDisposable
{
public const string MeterName = "StellaOps.Excititor.Attestation";
public const string MeterVersion = "1.0";
private readonly Meter _meter;
private bool _disposed;
public VexAttestationMetrics()
{
_meter = new Meter(MeterName, MeterVersion);
VerifyTotal = _meter.CreateCounter<long>("excititor.attestation.verify_total", description: "Attestation verification attempts grouped by result/component/rekor.");
VerifyDuration = _meter.CreateHistogram<double>("excititor.attestation.verify_duration_seconds", unit: "s", description: "Attestation verification latency in seconds.");
}
public Counter<long> VerifyTotal { get; }
public Histogram<double> VerifyDuration { get; }
public void Dispose()
{
if (_disposed)
{
return;
}
_meter.Dispose();
_disposed = true;
}
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Excititor.Attestation.Verification;
public sealed class VexAttestationVerificationOptions
{
private TimeSpan _maxClockSkew = TimeSpan.FromMinutes(5);
/// <summary>
/// When true, verification fails if no transparency record is present.
/// </summary>
public bool RequireTransparencyLog { get; set; } = true;
/// <summary>
/// Allows verification to succeed when the transparency log cannot be reached.
/// A diagnostic entry is still emitted to signal the degraded state.
/// </summary>
public bool AllowOfflineTransparency { get; set; }
/// <summary>
/// Maximum tolerated clock skew between the attestation creation time and the verification context timestamp.
/// </summary>
public TimeSpan MaxClockSkew
{
get => _maxClockSkew;
set => _maxClockSkew = value < TimeSpan.Zero ? TimeSpan.Zero : value;
}
/// <summary>
/// When true, DSSE signatures must verify successfully against configured trusted signers.
/// </summary>
public bool RequireSignatureVerification { get; set; }
/// <summary>
/// Mapping of trusted signer key identifiers to verification configuration.
/// </summary>
public ImmutableDictionary<string, TrustedSignerOptions> TrustedSigners { get; set; } =
ImmutableDictionary<string, TrustedSignerOptions>.Empty;
public sealed record TrustedSignerOptions
{
public string Algorithm { get; init; } = StellaOps.Cryptography.SignatureAlgorithms.Ed25519;
/// <summary>
/// Key identifier to resolve from the crypto provider registry.
/// </summary>
public string KeyReference { get; init; } = string.Empty;
/// <summary>
/// Optional provider hint to bias registry resolution.
/// </summary>
public string? ProviderHint { get; init; }
}
}

View File

@@ -0,0 +1,633 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Cryptography;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.Core;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation.Verification;
internal sealed class VexAttestationVerifier : IVexAttestationVerifier
{
private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions StatementSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) },
};
private readonly ILogger<VexAttestationVerifier> _logger;
private readonly ITransparencyLogClient? _transparencyLogClient;
private readonly VexAttestationVerificationOptions _options;
private readonly VexAttestationMetrics _metrics;
private readonly ICryptoProviderRegistry? _cryptoRegistry;
private readonly ImmutableDictionary<string, VexAttestationVerificationOptions.TrustedSignerOptions> _trustedSigners;
public VexAttestationVerifier(
ILogger<VexAttestationVerifier> logger,
ITransparencyLogClient? transparencyLogClient,
IOptions<VexAttestationVerificationOptions> options,
VexAttestationMetrics metrics,
ICryptoProviderRegistry? cryptoRegistry = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
ArgumentNullException.ThrowIfNull(options);
_transparencyLogClient = transparencyLogClient;
_options = options.Value;
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_cryptoRegistry = cryptoRegistry;
_trustedSigners = _options.TrustedSigners;
}
public async ValueTask<VexAttestationVerification> VerifyAsync(
VexAttestationVerificationRequest request,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var stopwatch = Stopwatch.StartNew();
var diagnostics = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
var resultLabel = "valid";
var rekorState = "skipped";
var component = request.IsReverify ? "worker" : "webservice";
void SetFailure(string reason) => diagnostics["failure_reason"] = reason;
using var activity = VexAttestationActivitySource.Value.StartActivity("Verify", ActivityKind.Internal);
activity?.SetTag("attestation.component", component);
activity?.SetTag("attestation.export_id", request.Attestation.ExportId);
try
{
if (string.IsNullOrWhiteSpace(request.Envelope))
{
diagnostics["envelope.state"] = "missing";
SetFailure("missing_envelope");
_logger.LogWarning("Attestation envelope is missing for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!TryDeserializeEnvelope(request.Envelope, out var envelope, diagnostics))
{
SetFailure("invalid_envelope");
_logger.LogWarning("Failed to deserialize attestation envelope for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!string.Equals(envelope.PayloadType, VexDsseBuilder.PayloadType, StringComparison.OrdinalIgnoreCase))
{
diagnostics["payload.type"] = envelope.PayloadType ?? string.Empty;
SetFailure("unexpected_payload_type");
_logger.LogWarning(
"Unexpected DSSE payload type {PayloadType} for export {ExportId}",
envelope.PayloadType,
request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (envelope.Signatures is null || envelope.Signatures.Count == 0)
{
diagnostics["signature.state"] = "missing";
SetFailure("missing_signature");
_logger.LogWarning("Attestation envelope for export {ExportId} does not contain signatures.", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
var payloadBase64 = envelope.Payload ?? string.Empty;
if (!TryDecodePayload(payloadBase64, out var payloadBytes, diagnostics))
{
SetFailure("payload_decode_failed");
_logger.LogWarning("Failed to decode attestation payload for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!TryDeserializeStatement(payloadBytes, out var statement, diagnostics))
{
SetFailure("invalid_statement");
_logger.LogWarning("Failed to deserialize DSSE statement for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidatePredicateType(statement, request, diagnostics))
{
SetFailure("predicate_type_mismatch");
_logger.LogWarning("Predicate type mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidateSubject(statement, request, diagnostics))
{
SetFailure("subject_mismatch");
_logger.LogWarning("Subject mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidatePredicate(statement, request, diagnostics))
{
SetFailure("predicate_mismatch");
_logger.LogWarning("Predicate payload mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidateMetadataDigest(envelope, request.Metadata, diagnostics))
{
SetFailure("envelope_digest_mismatch");
_logger.LogWarning("Attestation digest mismatch for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
if (!ValidateSignedAt(request.Metadata, request.Attestation.CreatedAt, diagnostics))
{
SetFailure("signedat_out_of_range");
_logger.LogWarning("SignedAt validation failed for export {ExportId}", request.Attestation.ExportId);
resultLabel = "invalid";
return BuildResult(false);
}
rekorState = await VerifyTransparencyAsync(request.Metadata, diagnostics, cancellationToken).ConfigureAwait(false);
if (rekorState is "missing" or "unverified" or "client_unavailable" or "unreachable")
{
SetFailure(rekorState);
resultLabel = "invalid";
return BuildResult(false);
}
var signaturesVerified = await VerifySignaturesAsync(payloadBytes, envelope.Signatures, diagnostics, cancellationToken).ConfigureAwait(false);
if (!signaturesVerified)
{
diagnostics["failure_reason"] = diagnostics.TryGetValue("signature.reason", out var reason)
? reason
: "signature_verification_failed";
if (_options.RequireSignatureVerification)
{
resultLabel = "invalid";
return BuildResult(false);
}
resultLabel = "degraded";
}
if (rekorState is "offline" && resultLabel != "invalid")
{
resultLabel = "degraded";
}
return BuildResult(true);
}
catch (Exception ex)
{
diagnostics["error"] = ex.GetType().Name;
diagnostics["error.message"] = ex.Message; resultLabel = "error";
_logger.LogError(ex, "Unexpected exception verifying attestation for export {ExportId}", request.Attestation.ExportId);
diagnostics["failure_reason"] = diagnostics.TryGetValue("error", out var errorCode)
? errorCode
: ex.GetType().Name;
return BuildResult(false);
}
finally
{
stopwatch.Stop();
var tags = new KeyValuePair<string, object?>[]
{
new("result", resultLabel),
new("component", component),
new("rekor", rekorState),
};
_metrics.VerifyTotal.Add(1, tags);
_metrics.VerifyDuration.Record(stopwatch.Elapsed.TotalSeconds, tags);
}
VexAttestationVerification BuildResult(bool isValid)
{
diagnostics["result"] = resultLabel;
diagnostics["component"] = component;
diagnostics["rekor.state"] = rekorState;
var snapshot = VexAttestationDiagnostics.FromBuilder(diagnostics);
if (activity is { } currentActivity)
{
currentActivity.SetTag("attestation.result", resultLabel);
currentActivity.SetTag("attestation.rekor", rekorState);
if (!isValid)
{
var failure = snapshot.FailureReason ?? "verification_failed";
currentActivity.SetStatus(ActivityStatusCode.Error, failure);
currentActivity.SetTag("attestation.failure_reason", failure);
}
else
{
currentActivity.SetStatus(resultLabel is "degraded"
? ActivityStatusCode.Ok
: ActivityStatusCode.Ok);
}
}
return new VexAttestationVerification(isValid, snapshot);
}
}
private async ValueTask<bool> VerifySignaturesAsync(
ReadOnlyMemory<byte> payloadBytes,
IReadOnlyList<DsseSignature> signatures,
ImmutableDictionary<string, string>.Builder diagnostics,
CancellationToken cancellationToken)
{
if (signatures is null || signatures.Count == 0)
{
diagnostics["signature.state"] = "missing";
return false;
}
if (_trustedSigners.Count == 0)
{
diagnostics["signature.state"] = "skipped";
diagnostics["signature.reason"] = "trust_not_configured";
return true;
}
if (_cryptoRegistry is null)
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "registry_unavailable";
return false;
}
foreach (var signature in signatures)
{
if (string.IsNullOrWhiteSpace(signature.Signature))
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "empty_signature";
return false;
}
if (string.IsNullOrWhiteSpace(signature.KeyId))
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "missing_key_id";
return false;
}
if (!_trustedSigners.TryGetValue(signature.KeyId, out var signerOptions))
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "untrusted_key";
diagnostics["signature.keyId"] = signature.KeyId;
return false;
}
byte[] signatureBytes;
try
{
signatureBytes = Convert.FromBase64String(signature.Signature);
}
catch (FormatException)
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "invalid_signature_encoding";
diagnostics["signature.keyId"] = signature.KeyId;
return false;
}
CryptoSignerResolution resolution;
try
{
resolution = _cryptoRegistry.ResolveSigner(
CryptoCapability.Verification,
signerOptions.Algorithm,
new CryptoKeyReference(signerOptions.KeyReference, signerOptions.ProviderHint));
}
catch (Exception ex)
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "resolution_failed";
diagnostics["signature.error"] = ex.GetType().Name;
diagnostics["signature.keyId"] = signature.KeyId;
return false;
}
var verified = await resolution.Signer
.VerifyAsync(payloadBytes, signatureBytes, cancellationToken)
.ConfigureAwait(false);
if (!verified)
{
diagnostics["signature.state"] = "error";
diagnostics["signature.reason"] = "verification_failed";
diagnostics["signature.keyId"] = signature.KeyId;
return false;
}
}
diagnostics["signature.state"] = "verified";
return true;
}
private static bool TryDeserializeEnvelope(
string envelopeJson,
out DsseEnvelope envelope,
ImmutableDictionary<string, string>.Builder diagnostics)
{
try
{
envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson, EnvelopeSerializerOptions)
?? throw new InvalidOperationException("Envelope deserialized to null.");
return true;
}
catch (Exception ex)
{
diagnostics["envelope.error"] = ex.GetType().Name;
envelope = default!;
return false;
}
}
private static bool TryDecodePayload(
string payloadBase64,
out byte[] payloadBytes,
ImmutableDictionary<string, string>.Builder diagnostics)
{
try
{
payloadBytes = Convert.FromBase64String(payloadBase64);
return true;
}
catch (FormatException)
{
diagnostics["payload.base64"] = "invalid";
payloadBytes = Array.Empty<byte>();
return false;
}
}
private static bool TryDeserializeStatement(
byte[] payload,
out VexInTotoStatement statement,
ImmutableDictionary<string, string>.Builder diagnostics)
{
try
{
statement = JsonSerializer.Deserialize<VexInTotoStatement>(payload, StatementSerializerOptions)
?? throw new InvalidOperationException("Statement deserialized to null.");
return true;
}
catch (Exception ex)
{
diagnostics["payload.error"] = ex.GetType().Name;
statement = default!;
return false;
}
}
private static bool ValidatePredicateType(
VexInTotoStatement statement,
VexAttestationVerificationRequest request,
ImmutableDictionary<string, string>.Builder diagnostics)
{
var predicateType = statement.PredicateType ?? string.Empty;
if (!string.Equals(predicateType, request.Metadata.PredicateType, StringComparison.Ordinal))
{
diagnostics["predicate.type"] = predicateType;
return false;
}
return true;
}
private static bool ValidateSubject(
VexInTotoStatement statement,
VexAttestationVerificationRequest request,
ImmutableDictionary<string, string>.Builder diagnostics)
{
if (statement.Subject is null || statement.Subject.Count != 1)
{
diagnostics["subject.count"] = (statement.Subject?.Count ?? 0).ToString();
return false;
}
var subject = statement.Subject[0];
if (!string.Equals(subject.Name, request.Attestation.ExportId, StringComparison.Ordinal))
{
diagnostics["subject.name"] = subject.Name ?? string.Empty;
return false;
}
if (subject.Digest is null)
{
diagnostics["subject.digest"] = "missing";
return false;
}
var algorithmKey = request.Attestation.Artifact.Algorithm.ToLowerInvariant();
if (!subject.Digest.TryGetValue(algorithmKey, out var digest)
|| !string.Equals(digest, request.Attestation.Artifact.Digest, StringComparison.OrdinalIgnoreCase))
{
diagnostics["subject.digest"] = digest ?? string.Empty;
return false;
}
return true;
}
private bool ValidatePredicate(
VexInTotoStatement statement,
VexAttestationVerificationRequest request,
ImmutableDictionary<string, string>.Builder diagnostics)
{
var predicate = statement.Predicate;
if (predicate is null)
{
diagnostics["predicate.state"] = "missing";
return false;
}
if (!string.Equals(predicate.ExportId, request.Attestation.ExportId, StringComparison.Ordinal))
{
diagnostics["predicate.exportId"] = predicate.ExportId ?? string.Empty;
return false;
}
if (!string.Equals(predicate.QuerySignature, request.Attestation.QuerySignature.Value, StringComparison.Ordinal))
{
diagnostics["predicate.querySignature"] = predicate.QuerySignature ?? string.Empty;
return false;
}
if (!string.Equals(predicate.ArtifactAlgorithm, request.Attestation.Artifact.Algorithm, StringComparison.OrdinalIgnoreCase)
|| !string.Equals(predicate.ArtifactDigest, request.Attestation.Artifact.Digest, StringComparison.OrdinalIgnoreCase))
{
diagnostics["predicate.artifact"] = $"{predicate.ArtifactAlgorithm}:{predicate.ArtifactDigest}";
return false;
}
if (predicate.Format != request.Attestation.Format)
{
diagnostics["predicate.format"] = predicate.Format.ToString();
return false;
}
var createdDelta = (predicate.CreatedAt - request.Attestation.CreatedAt).Duration();
if (createdDelta > _options.MaxClockSkew)
{
diagnostics["predicate.createdAtDelta"] = createdDelta.ToString();
return false;
}
if (!SetEquals(predicate.SourceProviders, request.Attestation.SourceProviders))
{
diagnostics["predicate.sourceProviders"] = string.Join(",", predicate.SourceProviders ?? Array.Empty<string>());
return false;
}
if (request.Attestation.Metadata.Count > 0)
{
if (predicate.Metadata is null)
{
diagnostics["predicate.metadata"] = "missing";
return false;
}
foreach (var kvp in request.Attestation.Metadata)
{
if (!predicate.Metadata.TryGetValue(kvp.Key, out var actual)
|| !string.Equals(actual, kvp.Value, StringComparison.Ordinal))
{
diagnostics[$"predicate.metadata.{kvp.Key}"] = actual ?? string.Empty;
return false;
}
}
}
return true;
}
private bool ValidateMetadataDigest(
DsseEnvelope envelope,
VexAttestationMetadata metadata,
ImmutableDictionary<string, string>.Builder diagnostics)
{
if (string.IsNullOrWhiteSpace(metadata.EnvelopeDigest))
{
diagnostics["metadata.envelopeDigest"] = "missing";
return false;
}
var computed = VexDsseBuilder.ComputeEnvelopeDigest(envelope);
if (!string.Equals(computed, metadata.EnvelopeDigest, StringComparison.OrdinalIgnoreCase))
{
diagnostics["metadata.envelopeDigest"] = metadata.EnvelopeDigest;
diagnostics["metadata.envelopeDigest.computed"] = computed;
return false;
}
diagnostics["metadata.envelopeDigest"] = "match";
return true;
}
private bool ValidateSignedAt(
VexAttestationMetadata metadata,
DateTimeOffset createdAt,
ImmutableDictionary<string, string>.Builder diagnostics)
{
if (metadata.SignedAt is null)
{
diagnostics["metadata.signedAt"] = "missing";
return false;
}
var delta = (metadata.SignedAt.Value - createdAt).Duration();
if (delta > _options.MaxClockSkew)
{
diagnostics["metadata.signedAtDelta"] = delta.ToString();
return false;
}
return true;
}
private async ValueTask<string> VerifyTransparencyAsync(
VexAttestationMetadata metadata,
ImmutableDictionary<string, string>.Builder diagnostics,
CancellationToken cancellationToken)
{
if (metadata.Rekor is null)
{
if (_options.RequireTransparencyLog)
{
diagnostics["rekor.state"] = "missing";
return "missing";
}
diagnostics["rekor.state"] = "disabled";
return "disabled";
}
if (_transparencyLogClient is null)
{
diagnostics["rekor.state"] = "client_unavailable";
return _options.RequireTransparencyLog ? "client_unavailable" : "disabled";
}
try
{
var verified = await _transparencyLogClient.VerifyAsync(metadata.Rekor.Location, cancellationToken).ConfigureAwait(false);
diagnostics["rekor.state"] = verified ? "verified" : "unverified";
return verified ? "verified" : "unverified";
}
catch (Exception ex)
{
diagnostics["rekor.error"] = ex.GetType().Name;
if (_options.AllowOfflineTransparency)
{
diagnostics["rekor.state"] = "offline";
return "offline";
}
diagnostics["rekor.state"] = "unreachable";
return "unreachable";
}
}
private static bool SetEquals(IReadOnlyCollection<string>? left, ImmutableArray<string> right)
{
if (left is null || left.Count == 0)
{
return right.IsDefaultOrEmpty || right.Length == 0;
}
if (right.IsDefaultOrEmpty)
{
return false;
}
var leftSet = new HashSet<string>(left, StringComparer.Ordinal);
var rightSet = new HashSet<string>(right, StringComparer.Ordinal);
return leftSet.SetEquals(rightSet);
}
}

View File

@@ -0,0 +1,112 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Attestation.Dsse;
using StellaOps.Excititor.Attestation.Models;
using StellaOps.Excititor.Attestation.Signing;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Core;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Attestation;
public sealed class VexAttestationClientOptions
{
public IReadOnlyDictionary<string, string> DefaultMetadata { get; set; } = ImmutableDictionary<string, string>.Empty;
}
public sealed class VexAttestationClient : IVexAttestationClient
{
private readonly VexDsseBuilder _builder;
private readonly ILogger<VexAttestationClient> _logger;
private readonly TimeProvider _timeProvider;
private readonly IReadOnlyDictionary<string, string> _defaultMetadata;
private readonly ITransparencyLogClient? _transparencyLogClient;
private readonly IVexAttestationVerifier _verifier;
public VexAttestationClient(
VexDsseBuilder builder,
IOptions<VexAttestationClientOptions> options,
ILogger<VexAttestationClient> logger,
IVexAttestationVerifier verifier,
TimeProvider? timeProvider = null,
ITransparencyLogClient? transparencyLogClient = null)
{
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
ArgumentNullException.ThrowIfNull(options);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_verifier = verifier ?? throw new ArgumentNullException(nameof(verifier));
_timeProvider = timeProvider ?? TimeProvider.System;
_defaultMetadata = options.Value.DefaultMetadata;
_transparencyLogClient = transparencyLogClient;
}
public async ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var mergedMetadata = MergeMetadata(request.Metadata, _defaultMetadata);
var envelope = await _builder.CreateEnvelopeAsync(request, mergedMetadata, cancellationToken).ConfigureAwait(false);
var envelopeDigest = VexDsseBuilder.ComputeEnvelopeDigest(envelope);
var signedAt = _timeProvider.GetUtcNow();
var diagnosticsBuilder = ImmutableDictionary<string, string>.Empty
.Add("envelope", JsonSerializer.Serialize(envelope))
.Add("predicateType", "https://stella-ops.org/attestations/vex-export");
VexRekorReference? rekorReference = null;
if (_transparencyLogClient is not null)
{
try
{
var entry = await _transparencyLogClient.SubmitAsync(envelope, cancellationToken).ConfigureAwait(false);
rekorReference = new VexRekorReference("0.2", entry.Location, entry.LogIndex, entry.InclusionProofUrl is not null ? new Uri(entry.InclusionProofUrl) : null);
diagnosticsBuilder = diagnosticsBuilder.Add("rekorLocation", entry.Location);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to submit attestation to Rekor transparency log");
throw;
}
}
var metadata = new VexAttestationMetadata(
predicateType: "https://stella-ops.org/attestations/vex-export",
rekor: rekorReference,
envelopeDigest: envelopeDigest,
signedAt: signedAt);
_logger.LogInformation("Generated DSSE envelope for export {ExportId} ({Digest})", request.ExportId, envelopeDigest);
return new VexAttestationResponse(metadata, diagnosticsBuilder);
}
public ValueTask<VexAttestationVerification> VerifyAsync(
VexAttestationVerificationRequest request,
CancellationToken cancellationToken)
=> _verifier.VerifyAsync(request, cancellationToken);
private static IReadOnlyDictionary<string, string> MergeMetadata(
IReadOnlyDictionary<string, string> requestMetadata,
IReadOnlyDictionary<string, string> defaults)
{
if (defaults.Count == 0)
{
return requestMetadata;
}
var merged = new Dictionary<string, string>(defaults, StringComparer.Ordinal);
foreach (var kvp in requestMetadata)
{
merged[kvp.Key] = kvp.Value;
}
return merged.ToImmutableDictionary(StringComparer.Ordinal);
}
}

View File

@@ -0,0 +1,33 @@
# AGENTS
## Role
Defines shared connector infrastructure for Excititor, including base contexts, result contracts, configuration binding, and helper utilities reused by all connector plug-ins.
## Scope
- `IVexConnector` context implementation, raw store helpers, verification hooks, and telemetry utilities.
- Configuration primitives (YAML parsing, secrets handling guidelines) and options validation.
- Connector lifecycle helpers for retries, paging, `.well-known` discovery, and resume markers.
- Documentation for connector packaging, plugin manifest metadata, and DI registration (see `docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md` and `docs/dev/templates/excititor-connector/`).
## Participants
- All Excititor connector projects reference this module to obtain base classes and context services.
- WebService/Worker instantiate connectors via plugin loader leveraging abstractions defined here.
## Interfaces & contracts
- Connector context, result, and telemetry interfaces; `VexConnectorDescriptor`, `VexConnectorBase`, options binder/validators, authentication helpers.
- Utility classes for HTTP clients, throttling, and deterministic logging.
## In/Out of scope
In: shared abstractions, helper utilities, configuration binding, documentation for connector authors.
Out: provider-specific logic (implemented in individual connector modules), storage persistence, HTTP host code.
## Observability & security expectations
- Provide structured logging helpers, correlation IDs, and metrics instrumentation toggles for connectors.
- Enforce redaction of secrets in logs and config dumps.
## Tests
- Abstraction/unit tests will live in `../StellaOps.Excititor.Connectors.Abstractions.Tests`, covering default behaviors and sample harness.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,12 @@
using System.Collections.Generic;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Custom validator hook executed after connector options are bound.
/// </summary>
/// <typeparam name="TOptions">Connector-specific options type.</typeparam>
public interface IVexConnectorOptionsValidator<in TOptions>
{
void Validate(VexConnectorDescriptor descriptor, TOptions options, IList<string> errors);
}

View File

@@ -0,0 +1,6 @@
using StellaOps.Plugin.Versioning;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Connectors.Abstractions.Tests")]
[assembly: StellaPluginVersion("1.0.0", MinimumHostVersion = "1.0.0", MaximumHostVersion = "1.99.99")]

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Plugin\StellaOps.Plugin.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,8 @@
# Completed Tasks
|EXCITITOR-CONN-ABS-01-001 Connector context & base classes|Team Excititor Connectors|EXCITITOR-CORE-01-003|**DONE (2025-10-17)** Added `StellaOps.Excititor.Connectors.Abstractions` project with `VexConnectorBase`, deterministic logging scopes, metadata builder helpers, and connector descriptors; docs updated to highlight the shared abstractions.|
|EXCITITOR-CONN-ABS-01-002 YAML options & validation|Team Excititor Connectors|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Delivered `VexConnectorOptionsBinder` + binder options/validators, environment-variable expansion, data-annotation checks, and custom validation hooks with documentation updates covering the workflow.|
|EXCITITOR-CONN-ABS-01-003 Plugin packaging & docs|Team Excititor Connectors|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Authored `docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md`, added quick-start template under `docs/dev/templates/excititor-connector/`, and updated module docs to reference the packaging workflow.|

View File

@@ -0,0 +1,10 @@
# Excititor Connectors Abstractions Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0297-M | DONE | Revalidated 2026-01-07. |
| AUDIT-0297-T | DONE | Revalidated 2026-01-07. |
| AUDIT-0297-A | TODO | Revalidated 2026-01-07 (open findings; pending approval). |

View File

@@ -0,0 +1,205 @@
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
namespace StellaOps.Excititor.Connectors.Abstractions.Trust;
public sealed record ConnectorSignerMetadataSet(
string SchemaVersion,
DateTimeOffset GeneratedAt,
ImmutableArray<ConnectorSignerMetadata> Connectors)
{
private readonly ImmutableDictionary<string, ConnectorSignerMetadata> _byId =
Connectors.ToImmutableDictionary(x => x.ConnectorId, StringComparer.OrdinalIgnoreCase);
public bool TryGet(string connectorId, [NotNullWhen(true)] out ConnectorSignerMetadata? metadata)
=> _byId.TryGetValue(connectorId, out metadata);
}
public sealed record ConnectorSignerMetadata(
string ConnectorId,
string ProviderName,
string ProviderSlug,
string IssuerTier,
ImmutableArray<ConnectorSignerSigner> Signers,
ConnectorSignerBundleRef? Bundle,
string? ValidFrom,
string? ValidTo,
bool Revoked,
string? Notes);
public sealed record ConnectorSignerSigner(
string Usage,
ImmutableArray<ConnectorSignerFingerprint> Fingerprints,
string? KeyLocator,
ImmutableArray<string> CertificateChain);
public sealed record ConnectorSignerFingerprint(
string Alg,
string Format,
string Value);
public sealed record ConnectorSignerBundleRef(
string Kind,
string Uri,
string? Digest,
DateTimeOffset? PublishedAt);
public static class ConnectorSignerMetadataLoader
{
public static ConnectorSignerMetadataSet? TryLoad(string? path, Stream? overrideStream = null)
{
if (string.IsNullOrWhiteSpace(path) && overrideStream is null)
{
return null;
}
try
{
using var stream = overrideStream ?? File.OpenRead(path!);
var root = JsonNode.Parse(stream, new JsonNodeOptions { PropertyNameCaseInsensitive = true });
if (root is null)
{
return null;
}
var version = root["schemaVersion"]?.GetValue<string?>() ?? "0.0.0";
var generatedAt = root["generatedAt"]?.GetValue<DateTimeOffset?>() ?? DateTimeOffset.MinValue;
var connectorsNode = root["connectors"] as JsonArray;
if (connectorsNode is null || connectorsNode.Count == 0)
{
return null;
}
var connectors = connectorsNode
.Select(ParseConnector)
.Where(c => c is not null)
.Select(c => c!)
.OrderBy(c => c.ConnectorId, StringComparer.Ordinal)
.ToImmutableArray();
return new ConnectorSignerMetadataSet(version, generatedAt, connectors);
}
catch
{
return null;
}
}
private static ConnectorSignerMetadata? ParseConnector(JsonNode? node)
{
if (node is not JsonObject obj)
{
return null;
}
var id = obj["connectorId"]?.GetValue<string?>();
var providerName = obj["provider"]?["name"]?.GetValue<string?>();
var providerSlug = obj["provider"]?["slug"]?.GetValue<string?>();
var issuerTier = obj["issuerTier"]?.GetValue<string?>();
var signers = (obj["signers"] as JsonArray)?.Select(ParseSigner)
.Where(x => x is not null)
.Select(x => x!)
.ToImmutableArray() ?? ImmutableArray<ConnectorSignerSigner>.Empty;
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(providerName) || signers.Length == 0)
{
return null;
}
return new ConnectorSignerMetadata(
id!,
providerName!,
providerSlug ?? providerName!,
issuerTier ?? "untrusted",
signers,
ParseBundle(obj["bundle"]),
obj["validFrom"]?.GetValue<string?>(),
obj["validTo"]?.GetValue<string?>(),
obj["revoked"]?.GetValue<bool?>() ?? false,
obj["notes"]?.GetValue<string?>());
}
private static ConnectorSignerSigner? ParseSigner(JsonNode? node)
{
if (node is not JsonObject obj)
{
return null;
}
var usage = obj["usage"]?.GetValue<string?>();
var fps = (obj["fingerprints"] as JsonArray)?.Select(ParseFingerprint)
.Where(x => x is not null)
.Select(x => x!)
.ToImmutableArray() ?? ImmutableArray<ConnectorSignerFingerprint>.Empty;
if (string.IsNullOrWhiteSpace(usage) || fps.IsDefaultOrEmpty)
{
return null;
}
var chain = (obj["certificateChain"] as JsonArray)?.Select(x => x?.GetValue<string?>())
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v!)
.ToImmutableArray() ?? ImmutableArray<string>.Empty;
return new ConnectorSignerSigner(
usage!,
fps,
obj["keyLocator"]?.GetValue<string?>(),
chain);
}
private static ConnectorSignerFingerprint? ParseFingerprint(JsonNode? node)
{
if (node is not JsonObject obj)
{
return null;
}
var alg = obj["alg"]?.GetValue<string?>();
var format = obj["format"]?.GetValue<string?>();
var value = obj["value"]?.GetValue<string?>();
if (string.IsNullOrWhiteSpace(alg) || string.IsNullOrWhiteSpace(format) || string.IsNullOrWhiteSpace(value))
{
return null;
}
return new ConnectorSignerFingerprint(alg!, format!, value!);
}
private static ConnectorSignerBundleRef? ParseBundle(JsonNode? node)
{
if (node is not JsonObject obj)
{
return null;
}
var kind = obj["kind"]?.GetValue<string?>();
var uri = obj["uri"]?.GetValue<string?>();
if (string.IsNullOrWhiteSpace(kind) || string.IsNullOrWhiteSpace(uri))
{
return null;
}
DateTimeOffset? published = null;
if (obj["publishedAt"] is JsonNode publishedNode && publishedNode.GetValue<string?>() is { } publishedString)
{
if (DateTimeOffset.TryParse(publishedString, out var parsed))
{
published = parsed;
}
}
return new ConnectorSignerBundleRef(
kind!,
uri!,
obj["digest"]?.GetValue<string?>(),
published);
}
}

View File

@@ -0,0 +1,82 @@
using Microsoft.Extensions.Logging;
using System;
using System.Globalization;
using System.IO;
using System.Linq;
namespace StellaOps.Excititor.Connectors.Abstractions.Trust;
public static class ConnectorSignerMetadataEnricher
{
private static readonly object Sync = new();
private static ConnectorSignerMetadataSet? _cached;
private static string? _cachedPath;
private const string EnvVar = "STELLAOPS_CONNECTOR_SIGNER_METADATA_PATH";
public static void Enrich(
VexConnectorMetadataBuilder builder,
string connectorId,
ILogger? logger = null,
string? metadataPath = null)
{
ArgumentNullException.ThrowIfNull(builder);
if (string.IsNullOrWhiteSpace(connectorId)) return;
var path = metadataPath ?? Environment.GetEnvironmentVariable(EnvVar);
if (string.IsNullOrWhiteSpace(path))
{
return;
}
var metadata = LoadCached(path, logger);
if (metadata is null || !metadata.TryGet(connectorId, out var connector))
{
return;
}
builder
.Add("vex.provenance.trust.issuerTier", connector.IssuerTier)
.Add("vex.provenance.trust.signers", string.Join(';', connector.Signers.SelectMany(s => s.Fingerprints.Select(fp => fp.Value))))
.Add("vex.provenance.trust.provider", connector.ProviderSlug);
if (connector.Bundle is { } bundle)
{
builder
.Add("vex.provenance.bundle.kind", bundle.Kind)
.Add("vex.provenance.bundle.uri", bundle.Uri)
.Add("vex.provenance.bundle.digest", bundle.Digest)
.Add("vex.provenance.bundle.publishedAt", bundle.PublishedAt?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture));
}
}
private static ConnectorSignerMetadataSet? LoadCached(string path, ILogger? logger)
{
if (!string.Equals(path, _cachedPath, StringComparison.OrdinalIgnoreCase) || _cached is null)
{
lock (Sync)
{
if (!string.Equals(path, _cachedPath, StringComparison.OrdinalIgnoreCase) || _cached is null)
{
if (!File.Exists(path))
{
logger?.LogDebug("Connector signer metadata file not found at {Path}; skipping enrichment.", path);
_cached = null;
}
else
{
_cached = ConnectorSignerMetadataLoader.TryLoad(path);
_cachedPath = path;
if (_cached is null)
{
logger?.LogWarning("Failed to load connector signer metadata from {Path}.", path);
}
}
}
}
}
return _cached;
}
}

View File

@@ -0,0 +1,100 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
using System.Collections.Immutable;
using System.Security.Cryptography;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Convenience base class for implementing <see cref="IVexConnector" />.
/// </summary>
public abstract class VexConnectorBase : IVexConnector
{
protected VexConnectorBase(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider? timeProvider = null)
{
Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
TimeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
public string Id => Descriptor.Id;
/// <inheritdoc />
public VexProviderKind Kind => Descriptor.Kind;
public VexConnectorDescriptor Descriptor { get; }
protected ILogger Logger { get; }
protected TimeProvider TimeProvider { get; }
protected DateTimeOffset UtcNow() => TimeProvider.GetUtcNow();
protected VexRawDocument CreateRawDocument(
VexDocumentFormat format,
Uri sourceUri,
ReadOnlyMemory<byte> content,
ImmutableDictionary<string, string>? metadata = null)
{
if (sourceUri is null)
{
throw new ArgumentNullException(nameof(sourceUri));
}
var digest = ComputeSha256(content.Span);
var captured = TimeProvider.GetUtcNow();
return new VexRawDocument(
Descriptor.Id,
format,
sourceUri,
captured,
digest,
content,
metadata ?? ImmutableDictionary<string, string>.Empty);
}
protected IDisposable BeginConnectorScope(string operation, IReadOnlyDictionary<string, object?>? metadata = null)
=> VexConnectorLogScope.Begin(Logger, Descriptor, operation, metadata);
protected void LogConnectorEvent(LogLevel level, string eventName, string message, IReadOnlyDictionary<string, object?>? metadata = null, Exception? exception = null)
{
using var scope = BeginConnectorScope(eventName, metadata);
if (exception is null)
{
Logger.Log(level, "{Message}", message);
}
else
{
Logger.Log(level, exception, "{Message}", message);
}
}
protected ImmutableDictionary<string, string> BuildMetadata(Action<VexConnectorMetadataBuilder> configure)
{
ArgumentNullException.ThrowIfNull(configure);
var builder = new VexConnectorMetadataBuilder();
configure(builder);
return builder.Build();
}
private static string ComputeSha256(ReadOnlySpan<byte> content)
{
Span<byte> buffer = stackalloc byte[32];
if (SHA256.TryHashData(content, buffer, out _))
{
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
}
using var sha = SHA256.Create();
var hash = sha.ComputeHash(content.ToArray());
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
}
public abstract ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken);
public abstract IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken);
public abstract ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,55 @@
using StellaOps.Excititor.Core;
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Static descriptor for a Excititor connector plug-in.
/// </summary>
public sealed record VexConnectorDescriptor
{
public VexConnectorDescriptor(string id, VexProviderKind kind, string displayName)
{
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentException("Connector id must be provided.", nameof(id));
}
Id = id;
Kind = kind;
DisplayName = string.IsNullOrWhiteSpace(displayName) ? id : displayName;
}
/// <summary>
/// Stable connector identifier (matches provider id).
/// </summary>
public string Id { get; }
/// <summary>
/// Provider kind served by the connector.
/// </summary>
public VexProviderKind Kind { get; }
/// <summary>
/// Human friendly name used in logs/diagnostics.
/// </summary>
public string DisplayName { get; }
/// <summary>
/// Optional friendly description.
/// </summary>
public string? Description { get; init; }
/// <summary>
/// Document formats the connector is expected to emit.
/// </summary>
public ImmutableArray<VexDocumentFormat> SupportedFormats { get; init; } = ImmutableArray<VexDocumentFormat>.Empty;
/// <summary>
/// Optional tags surfaced in diagnostics (e.g. "beta", "offline").
/// </summary>
public ImmutableArray<string> Tags { get; init; } = ImmutableArray<string>.Empty;
public override string ToString() => $"{Id} ({Kind})";
}

View File

@@ -0,0 +1,51 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
using System.Linq;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Helper to establish deterministic logging scopes for connector operations.
/// </summary>
public static class VexConnectorLogScope
{
public static IDisposable Begin(ILogger logger, VexConnectorDescriptor descriptor, string operation, IReadOnlyDictionary<string, object?>? metadata = null)
{
ArgumentNullException.ThrowIfNull(logger);
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentException.ThrowIfNullOrEmpty(operation);
var scopeValues = new List<KeyValuePair<string, object?>>
{
new("vex.connector.id", descriptor.Id),
new("vex.connector.kind", descriptor.Kind.ToString()),
new("vex.connector.operation", operation),
};
if (!string.Equals(descriptor.DisplayName, descriptor.Id, StringComparison.Ordinal))
{
scopeValues.Add(new KeyValuePair<string, object?>("vex.connector.displayName", descriptor.DisplayName));
}
if (!string.IsNullOrWhiteSpace(descriptor.Description))
{
scopeValues.Add(new KeyValuePair<string, object?>("vex.connector.description", descriptor.Description));
}
if (!descriptor.Tags.IsDefaultOrEmpty)
{
scopeValues.Add(new KeyValuePair<string, object?>("vex.connector.tags", string.Join(",", descriptor.Tags)));
}
if (metadata is not null)
{
foreach (var kvp in metadata.OrderBy(static pair => pair.Key, StringComparer.Ordinal))
{
scopeValues.Add(new KeyValuePair<string, object?>($"vex.{kvp.Key}", kvp.Value));
}
}
return logger.BeginScope(scopeValues)!;
}
}

View File

@@ -0,0 +1,38 @@
using System.Collections.Immutable;
using System.Globalization;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Builds deterministic metadata dictionaries for raw documents and logging scopes.
/// </summary>
public sealed class VexConnectorMetadataBuilder
{
private readonly SortedDictionary<string, string> _values = new(StringComparer.Ordinal);
public VexConnectorMetadataBuilder Add(string key, string? value)
{
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
{
_values[key] = value!;
}
return this;
}
public VexConnectorMetadataBuilder Add(string key, DateTimeOffset value)
=> Add(key, value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture));
public VexConnectorMetadataBuilder AddRange(IEnumerable<KeyValuePair<string, string?>> items)
{
foreach (var item in items)
{
Add(item.Key, item.Value);
}
return this;
}
public ImmutableDictionary<string, string> Build()
=> _values.ToImmutableDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal);
}

View File

@@ -0,0 +1,158 @@
using Microsoft.Extensions.Configuration;
using StellaOps.Excititor.Core;
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Linq;
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Provides strongly typed binding and validation for connector options.
/// </summary>
public static class VexConnectorOptionsBinder
{
public static TOptions Bind<TOptions>(
VexConnectorDescriptor descriptor,
VexConnectorSettings settings,
VexConnectorOptionsBinderOptions? options = null,
IEnumerable<IVexConnectorOptionsValidator<TOptions>>? validators = null)
where TOptions : class, new()
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(settings);
var binderSettings = options ?? new VexConnectorOptionsBinderOptions();
var transformed = TransformValues(settings, binderSettings);
var configuration = BuildConfiguration(transformed);
var result = new TOptions();
var errors = new List<string>();
try
{
configuration.Bind(
result,
binderOptions => binderOptions.ErrorOnUnknownConfiguration = !binderSettings.AllowUnknownKeys);
}
catch (InvalidOperationException ex) when (!binderSettings.AllowUnknownKeys)
{
errors.Add(ex.Message);
}
binderSettings.PostConfigure?.Invoke(result);
if (binderSettings.ValidateDataAnnotations)
{
ValidateDataAnnotations(result, errors);
}
if (validators is not null)
{
foreach (var validator in validators)
{
validator?.Validate(descriptor, result, errors);
}
}
if (errors.Count > 0)
{
throw new VexConnectorOptionsValidationException(descriptor.Id, errors);
}
return result;
}
private static ImmutableDictionary<string, string?> TransformValues(
VexConnectorSettings settings,
VexConnectorOptionsBinderOptions binderOptions)
{
var builder = ImmutableDictionary.CreateBuilder<string, string?>(StringComparer.OrdinalIgnoreCase);
foreach (var kvp in settings.Values)
{
var value = kvp.Value;
if (binderOptions.TrimWhitespace && value is not null)
{
value = value.Trim();
}
if (binderOptions.TreatEmptyAsNull && string.IsNullOrEmpty(value))
{
value = null;
}
if (value is not null && binderOptions.ExpandEnvironmentVariables)
{
value = Environment.ExpandEnvironmentVariables(value);
}
if (binderOptions.ValueTransformer is not null)
{
value = binderOptions.ValueTransformer.Invoke(kvp.Key, value);
}
builder[kvp.Key] = value;
}
return builder.ToImmutable();
}
private static IConfiguration BuildConfiguration(ImmutableDictionary<string, string?> values)
{
var sources = new List<KeyValuePair<string, string?>>();
foreach (var kvp in values)
{
if (kvp.Value is not null)
{
sources.Add(new KeyValuePair<string, string?>(kvp.Key, kvp.Value));
}
}
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.Add(new DictionaryConfigurationSource(sources));
return configurationBuilder.Build();
}
private static void ValidateDataAnnotations<TOptions>(TOptions options, IList<string> errors)
{
var validationResults = new List<ValidationResult>();
var validationContext = new ValidationContext(options!);
if (!Validator.TryValidateObject(options!, validationContext, validationResults, validateAllProperties: true))
{
foreach (var validationResult in validationResults)
{
if (!string.IsNullOrWhiteSpace(validationResult.ErrorMessage))
{
errors.Add(validationResult.ErrorMessage);
}
}
}
}
private sealed class DictionaryConfigurationSource : IConfigurationSource
{
private readonly IReadOnlyList<KeyValuePair<string, string?>> _data;
public DictionaryConfigurationSource(IEnumerable<KeyValuePair<string, string?>> data)
{
_data = data?.ToList() ?? new List<KeyValuePair<string, string?>>();
}
public IConfigurationProvider Build(IConfigurationBuilder builder) => new DictionaryConfigurationProvider(_data);
}
private sealed class DictionaryConfigurationProvider : ConfigurationProvider
{
public DictionaryConfigurationProvider(IEnumerable<KeyValuePair<string, string?>> data)
{
foreach (var pair in data)
{
if (pair.Value is not null)
{
Data[pair.Key] = pair.Value;
}
}
}
}
}

View File

@@ -0,0 +1,45 @@
namespace StellaOps.Excititor.Connectors.Abstractions;
/// <summary>
/// Customisation options for connector options binding.
/// </summary>
public sealed class VexConnectorOptionsBinderOptions
{
/// <summary>
/// Indicates whether environment variables should be expanded in option values.
/// Defaults to <c>true</c>.
/// </summary>
public bool ExpandEnvironmentVariables { get; set; } = true;
/// <summary>
/// When <c>true</c> the binder trims whitespace around option values.
/// </summary>
public bool TrimWhitespace { get; set; } = true;
/// <summary>
/// Converts empty strings to <c>null</c> before binding. Default: <c>true</c>.
/// </summary>
public bool TreatEmptyAsNull { get; set; } = true;
/// <summary>
/// When <c>false</c>, binding fails if unknown configuration keys are provided.
/// Default: <c>true</c> (permitting unknown keys).
/// </summary>
public bool AllowUnknownKeys { get; set; } = true;
/// <summary>
/// Enables <see cref="System.ComponentModel.DataAnnotations"/> validation after binding.
/// Default: <c>true</c>.
/// </summary>
public bool ValidateDataAnnotations { get; set; } = true;
/// <summary>
/// Optional post-configuration callback executed after binding.
/// </summary>
public Action<object>? PostConfigure { get; set; }
/// <summary>
/// Optional hook to transform raw configuration values before binding.
/// </summary>
public Func<string, string?, string?>? ValueTransformer { get; set; }
}

View File

@@ -0,0 +1,36 @@
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.Abstractions;
public sealed class VexConnectorOptionsValidationException : Exception
{
public VexConnectorOptionsValidationException(
string connectorId,
IEnumerable<string> errors)
: base(BuildMessage(connectorId, errors))
{
ConnectorId = connectorId;
Errors = errors?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
}
public string ConnectorId { get; }
public ImmutableArray<string> Errors { get; }
private static string BuildMessage(string connectorId, IEnumerable<string> errors)
{
var builder = new System.Text.StringBuilder();
builder.Append("Connector options validation failed for '");
builder.Append(connectorId);
builder.Append("'.");
var list = errors?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
if (!list.IsDefaultOrEmpty)
{
builder.Append(" Errors: ");
builder.Append(string.Join("; ", list));
}
return builder.ToString();
}
}

View File

@@ -0,0 +1,35 @@
# AGENTS
## Role
Connector responsible for ingesting Cisco CSAF VEX advisories and handing raw documents to normalizers with Cisco-specific metadata.
## Scope
- Discovery of Cisco CSAF collection endpoints, authentication (when required), and pagination routines.
- HTTP retries/backoff, checksum verification, and document deduplication before storage.
- Mapping Cisco advisory identifiers, product hierarchies, and severity hints into connector metadata.
- Surfacing provider trust configuration aligned with policy expectations.
## Participants
- Worker drives scheduled pulls; WebService may trigger manual runs.
- CSAF normalizer consumes raw documents to emit claims.
- Policy module references connector trust hints (e.g., Cisco signing identities).
## Interfaces & contracts
- Implements `IVexConnector` using shared abstractions for HTTP/resume handling.
- Provides options for API tokens, rate limits, and concurrency.
## In/Out of scope
In: data fetching, provider metadata, retry controls, raw document persistence.
Out: normalization/export, attestation, Mongo wiring (handled in other modules).
Out: normalization/export, attestation, Postgres/in-memory wiring (handled in other modules).
## Observability & security expectations
- Log fetch batches with document counts/durations; mask credentials.
- Emit metrics for rate-limit hits, retries, and quarantine events.
## Tests
- Unit tests plus HTTP harness fixtures will live in `../StellaOps.Excititor.Connectors.Cisco.CSAF.Tests`.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,309 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF;
public sealed class CiscoCsafConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:cisco",
kind: VexProviderKind.Vendor,
displayName: "Cisco CSAF")
{
Tags = ImmutableArray.Create("cisco", "csaf"),
};
private readonly CiscoProviderMetadataLoader _metadataLoader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>> _validators;
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
private CiscoConnectorOptions? _options;
private CiscoProviderMetadataResult? _providerMetadata;
public CiscoCsafConnector(
CiscoProviderMetadataLoader metadataLoader,
IHttpClientFactory httpClientFactory,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>>? validators,
ILogger<CiscoCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<CiscoConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary<string, object?>
{
["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length,
["fromOffline"] = _providerMetadata.FromOfflineSnapshot,
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_providerMetadata is null)
{
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
}
await UpsertProviderAsync(context.Services, _providerMetadata.Provider, cancellationToken).ConfigureAwait(false);
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue;
var latestTimestamp = state?.LastUpdated ?? since;
var stateChanged = false;
var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
foreach (var directory in _providerMetadata.Provider.BaseUris)
{
await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false))
{
var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue;
if (published <= since)
{
continue;
}
using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
contentResponse.EnsureSuccessStatusCode();
var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(builder =>
{
builder
.Add("cisco.csaf.advisoryId", advisory.Id)
.Add("cisco.csaf.revision", advisory.Revision)
.Add("cisco.csaf.published", advisory.Published?.ToString("O", CultureInfo.InvariantCulture))
.Add("cisco.csaf.modified", advisory.LastModified?.ToString("O", CultureInfo.InvariantCulture))
.Add("cisco.csaf.sha256", advisory.Sha256);
AddProvenanceMetadata(builder);
});
var rawDocument = CreateRawDocument(
VexDocumentFormat.Csaf,
advisory.DocumentUri,
payload,
metadata);
if (!digestSet.Add(rawDocument.Digest))
{
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
if (published > latestTimestamp)
{
latestTimestamp = published;
}
yield return rawDocument;
}
}
if (stateChanged)
{
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing.");
private async IAsyncEnumerable<CiscoAdvisoryEntry> EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var nextUri = BuildIndexUri(directory, null);
while (nextUri is not null)
{
using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var page = JsonSerializer.Deserialize<CiscoAdvisoryIndex>(json, _serializerOptions);
if (page?.Advisories is null)
{
yield break;
}
foreach (var advisory in page.Advisories)
{
if (string.IsNullOrWhiteSpace(advisory.Url))
{
continue;
}
if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri))
{
continue;
}
if (!documentUri.IsAbsoluteUri)
{
documentUri = new Uri(directory, documentUri);
}
yield return new CiscoAdvisoryEntry(
advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(),
documentUri,
advisory.Revision,
advisory.Published,
advisory.LastModified,
advisory.Sha256);
}
nextUri = ResolveNextUri(directory, page.Next);
}
}
private static Uri BuildIndexUri(Uri directory, string? relative)
{
if (string.IsNullOrWhiteSpace(relative))
{
var baseText = directory.ToString();
if (!baseText.EndsWith('/'))
{
baseText += "/";
}
return new Uri(new Uri(baseText, UriKind.Absolute), "index.json");
}
if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute))
{
return absolute;
}
var baseTextRelative = directory.ToString();
if (!baseTextRelative.EndsWith('/'))
{
baseTextRelative += "/";
}
return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative);
}
private static Uri? ResolveNextUri(Uri directory, string? next)
{
if (string.IsNullOrWhiteSpace(next))
{
return null;
}
return BuildIndexUri(directory, next);
}
private void AddProvenanceMetadata(VexConnectorMetadataBuilder builder)
{
if (_providerMetadata is null)
{
return;
}
var provider = _providerMetadata.Provider;
builder.Add("vex.provenance.provider", provider.Id);
builder.Add("vex.provenance.providerName", provider.DisplayName);
builder.Add("vex.provenance.trust.weight", provider.Trust.Weight.ToString("0.###", CultureInfo.InvariantCulture));
if (provider.Trust.Cosign is { } cosign)
{
builder.Add("vex.provenance.cosign.issuer", cosign.Issuer);
builder.Add("vex.provenance.cosign.identityPattern", cosign.IdentityPattern);
}
if (!provider.Trust.PgpFingerprints.IsDefaultOrEmpty && provider.Trust.PgpFingerprints.Length > 0)
{
builder.Add("vex.provenance.pgp.fingerprints", string.Join(",", provider.Trust.PgpFingerprints));
}
}
private static async ValueTask UpsertProviderAsync(IServiceProvider services, VexProvider provider, CancellationToken cancellationToken)
{
if (services is null)
{
return;
}
var store = services.GetService<IVexProviderStore>();
if (store is null)
{
return;
}
await store.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
}
private sealed record CiscoAdvisoryIndex
{
public List<CiscoAdvisory>? Advisories { get; init; }
public string? Next { get; init; }
}
private sealed record CiscoAdvisory
{
public string? Id { get; init; }
public string? Url { get; init; }
public string? Revision { get; init; }
public DateTimeOffset? Published { get; init; }
public DateTimeOffset? LastModified { get; init; }
public string? Sha256 { get; init; }
}
private sealed record CiscoAdvisoryEntry(
string Id,
Uri DocumentUri,
string? Revision,
DateTimeOffset? Published,
DateTimeOffset? LastModified,
string? Sha256);
}

View File

@@ -0,0 +1,58 @@
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
public sealed class CiscoConnectorOptions : IValidatableObject
{
public const string HttpClientName = "cisco-csaf";
/// <summary>
/// Endpoint for Cisco CSAF provider metadata discovery.
/// </summary>
[Required]
public string MetadataUri { get; set; } = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json";
/// <summary>
/// Optional bearer token used when Cisco endpoints require authentication.
/// </summary>
public string? ApiToken { get; set; }
/// <summary>
/// How long provider metadata remains cached.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6);
/// <summary>
/// Whether to prefer offline snapshots when fetching metadata.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// When set, provider metadata will be persisted to the given file path.
/// </summary>
public bool PersistOfflineSnapshot { get; set; }
public string? OfflineSnapshotPath { get; set; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (string.IsNullOrWhiteSpace(MetadataUri))
{
yield return new ValidationResult("MetadataUri must be provided.", new[] { nameof(MetadataUri) });
}
else if (!Uri.TryCreate(MetadataUri, UriKind.Absolute, out _))
{
yield return new ValidationResult("MetadataUri must be an absolute URI.", new[] { nameof(MetadataUri) });
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
yield return new ValidationResult("MetadataCacheDuration must be greater than zero.", new[] { nameof(MetadataCacheDuration) });
}
if (PersistOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
yield return new ValidationResult("OfflineSnapshotPath must be provided when PersistOfflineSnapshot is enabled.", new[] { nameof(OfflineSnapshotPath) });
}
}
}

View File

@@ -0,0 +1,26 @@
using StellaOps.Excititor.Connectors.Abstractions;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
public sealed class CiscoConnectorOptionsValidator : IVexConnectorOptionsValidator<CiscoConnectorOptions>
{
public void Validate(VexConnectorDescriptor descriptor, CiscoConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
var validationResults = new List<ValidationResult>();
if (!Validator.TryValidateObject(options, new ValidationContext(options), validationResults, validateAllProperties: true))
{
foreach (var result in validationResults)
{
errors.Add(result.ErrorMessage ?? "Cisco connector options validation failed.");
}
}
}
}

View File

@@ -0,0 +1,53 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
using StellaOps.Excititor.Core;
using System.ComponentModel.DataAnnotations;
using System.IO.Abstractions;
using System.Net.Http.Headers;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection;
public static class CiscoConnectorServiceCollectionExtensions
{
public static IServiceCollection AddCiscoCsafConnector(this IServiceCollection services, Action<CiscoConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.AddOptions<CiscoConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
})
.PostConfigure(options =>
{
Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true);
});
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.TryAddEnumerable(ServiceDescriptor.Singleton<IVexConnectorOptionsValidator<CiscoConnectorOptions>, CiscoConnectorOptionsValidator>());
services.AddHttpClient(CiscoConnectorOptions.HttpClientName)
.ConfigureHttpClient((provider, client) =>
{
var options = provider.GetRequiredService<IOptions<CiscoConnectorOptions>>().Value;
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
if (!string.IsNullOrWhiteSpace(options.ApiToken))
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiToken);
}
});
services.AddSingleton<CiscoProviderMetadataLoader>();
services.AddSingleton<IVexConnector, CiscoCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,338 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Core;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http.Headers;
using System.Text.Json;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
public sealed class CiscoProviderMetadataLoader
{
public const string CacheKey = "StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly ILogger<CiscoProviderMetadataLoader> _logger;
private readonly CiscoConnectorOptions _options;
private readonly IFileSystem _fileSystem;
private readonly TimeProvider _timeProvider;
private readonly JsonSerializerOptions _serializerOptions;
private readonly SemaphoreSlim _semaphore = new(1, 1);
public CiscoProviderMetadataLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IOptions<CiscoConnectorOptions> options,
ILogger<CiscoProviderMetadataLoader> logger,
IFileSystem? fileSystem = null,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value ?? throw new ArgumentNullException(nameof(options));
_fileSystem = fileSystem ?? new FileSystem();
_timeProvider = timeProvider ?? TimeProvider.System;
_serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
}
public async Task<CiscoProviderMetadataResult> LoadAsync(CancellationToken cancellationToken)
{
var now = _timeProvider.GetUtcNow();
if (_memoryCache.TryGetValue<CacheEntry>(CacheKey, out var cached) && cached is not null && !cached.IsExpired(now))
{
_logger.LogDebug("Returning cached Cisco provider metadata (expires {Expires}).", cached.ExpiresAt);
return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
now = _timeProvider.GetUtcNow();
if (_memoryCache.TryGetValue<CacheEntry>(CacheKey, out cached) && cached is not null && !cached.IsExpired(now))
{
return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true);
}
CacheEntry? previous = cached;
if (!_options.PreferOfflineSnapshot)
{
var network = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false);
if (network is not null)
{
StoreCache(network);
return new CiscoProviderMetadataResult(network.Provider, network.FetchedAt, false, false);
}
}
var offline = TryLoadFromOffline();
if (offline is not null)
{
var entry = offline with
{
FetchedAt = _timeProvider.GetUtcNow(),
ExpiresAt = _timeProvider.GetUtcNow() + _options.MetadataCacheDuration,
FromOffline = true,
};
StoreCache(entry);
return new CiscoProviderMetadataResult(entry.Provider, entry.FetchedAt, true, false);
}
throw new InvalidOperationException("Unable to load Cisco CSAF provider metadata from network or offline snapshot.");
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri);
if (!string.IsNullOrWhiteSpace(_options.ApiToken))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiToken);
}
if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag))
{
request.Headers.IfNoneMatch.Add(etag);
}
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotModified && previous is not null)
{
_logger.LogDebug("Cisco provider metadata not modified (etag {ETag}).", previous.ETag);
return previous with
{
FetchedAt = _timeProvider.GetUtcNow(),
ExpiresAt = _timeProvider.GetUtcNow() + _options.MetadataCacheDuration,
};
}
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var provider = ParseProvider(payload);
var etagHeader = response.Headers.ETag?.ToString();
if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
try
{
_fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
}
return new CacheEntry(
provider,
_timeProvider.GetUtcNow(),
_timeProvider.GetUtcNow() + _options.MetadataCacheDuration,
etagHeader,
FromOffline: false);
}
catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot)
{
_logger.LogWarning(ex, "Failed to fetch Cisco provider metadata from {Uri}; falling back to offline snapshot when available.", _options.MetadataUri);
return null;
}
}
private CacheEntry? TryLoadFromOffline()
{
if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath))
{
_logger.LogWarning("Cisco offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath);
var provider = ParseProvider(payload);
return new CacheEntry(provider, _timeProvider.GetUtcNow(), _timeProvider.GetUtcNow() + _options.MetadataCacheDuration, null, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Cisco provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath);
return null;
}
}
private VexProvider ParseProvider(string payload)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Cisco provider metadata payload was empty.");
}
ProviderMetadataDocument? document;
try
{
document = JsonSerializer.Deserialize<ProviderMetadataDocument>(payload, _serializerOptions);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Failed to parse Cisco provider metadata.", ex);
}
if (document?.Metadata?.Publisher?.ContactDetails is null || string.IsNullOrWhiteSpace(document.Metadata.Publisher.ContactDetails.Id))
{
throw new InvalidOperationException("Cisco provider metadata did not include a publisher identifier.");
}
var discovery = new VexProviderDiscovery(document.Discovery?.WellKnown, document.Discovery?.RolIe);
var trust = document.Trust is null
? VexProviderTrust.Default
: new VexProviderTrust(
document.Trust.Weight ?? 1.0,
document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty),
document.Trust.PgpFingerprints ?? Enumerable.Empty<string>());
var directories = document.Distributions?.Directories is null
? Enumerable.Empty<Uri>()
: document.Distributions.Directories
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Select(static s => Uri.TryCreate(s, UriKind.Absolute, out var uri) ? uri : null)
.Where(static uri => uri is not null)!
.Select(static uri => uri!);
return new VexProvider(
id: document.Metadata.Publisher.ContactDetails.Id,
displayName: document.Metadata.Publisher.Name ?? document.Metadata.Publisher.ContactDetails.Id,
kind: document.Metadata.Publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub,
baseUris: directories,
discovery: discovery,
trust: trust,
enabled: true);
}
private void StoreCache(CacheEntry entry)
{
var options = new MemoryCacheEntryOptions
{
AbsoluteExpiration = entry.ExpiresAt,
};
_memoryCache.Set(CacheKey, entry, options);
}
private sealed record CacheEntry(
VexProvider Provider,
DateTimeOffset FetchedAt,
DateTimeOffset ExpiresAt,
string? ETag,
bool FromOffline)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}
}
public sealed record CiscoProviderMetadataResult(
VexProvider Provider,
DateTimeOffset FetchedAt,
bool FromOfflineSnapshot,
bool ServedFromCache);
#region document models
internal sealed class ProviderMetadataDocument
{
[System.Text.Json.Serialization.JsonPropertyName("metadata")]
public ProviderMetadataMetadata Metadata { get; set; } = new();
[System.Text.Json.Serialization.JsonPropertyName("discovery")]
public ProviderMetadataDiscovery? Discovery { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("trust")]
public ProviderMetadataTrust? Trust { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("distributions")]
public ProviderMetadataDistributions? Distributions { get; set; }
}
internal sealed class ProviderMetadataMetadata
{
[System.Text.Json.Serialization.JsonPropertyName("publisher")]
public ProviderMetadataPublisher Publisher { get; set; } = new();
}
internal sealed class ProviderMetadataPublisher
{
[System.Text.Json.Serialization.JsonPropertyName("name")]
public string? Name { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("category")]
public string? Category { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("contact_details")]
public ProviderMetadataPublisherContact ContactDetails { get; set; } = new();
}
internal sealed class ProviderMetadataPublisherContact
{
[System.Text.Json.Serialization.JsonPropertyName("id")]
public string? Id { get; set; }
}
internal sealed class ProviderMetadataDiscovery
{
[System.Text.Json.Serialization.JsonPropertyName("well_known")]
public Uri? WellKnown { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("rolie")]
public Uri? RolIe { get; set; }
}
internal sealed class ProviderMetadataTrust
{
[System.Text.Json.Serialization.JsonPropertyName("weight")]
public double? Weight { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("cosign")]
public ProviderMetadataTrustCosign? Cosign { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("pgp_fingerprints")]
public string[]? PgpFingerprints { get; set; }
}
internal sealed class ProviderMetadataTrustCosign
{
[System.Text.Json.Serialization.JsonPropertyName("issuer")]
public string? Issuer { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("identity_pattern")]
public string? IdentityPattern { get; set; }
}
internal sealed class ProviderMetadataDistributions
{
[System.Text.Json.Serialization.JsonPropertyName("directories")]
public string[]? Directories { get; set; }
}
#endregion

View File

@@ -0,0 +1,6 @@
using StellaOps.Plugin.Versioning;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Connectors.Cisco.CSAF.Tests")]
[assembly: StellaPluginVersion("1.0.0", MinimumHostVersion = "1.0.0", MaximumHostVersion = "1.99.99")]

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Persistence\StellaOps.Excititor.Persistence.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="System.IO.Abstractions" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,8 @@
# Completed Tasks
|EXCITITOR-CONN-CISCO-01-001 Endpoint discovery & auth plumbing|Team Excititor Connectors Cisco|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added `CiscoProviderMetadataLoader` with bearer token support, offline snapshot fallback, DI helpers, and tests covering network/offline discovery to unblock subsequent fetch work.|
|EXCITITOR-CONN-CISCO-01-002 CSAF pull loop & pagination|Team Excititor Connectors Cisco|EXCITITOR-CONN-CISCO-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** Implemented paginated advisory fetch using provider directories, raw document persistence with dedupe/state tracking, offline resiliency, and unit coverage.|
|EXCITITOR-CONN-CISCO-01-003 Provider trust metadata|Team Excititor Connectors Cisco|EXCITITOR-CONN-CISCO-01-002, EXCITITOR-POLICY-01-001|**DONE (2025-10-29)** Connector now annotates raw documents with provider trust + cosign/PGP provenance and upserts `VexProvider` entries; new unit coverage asserts metadata emission and provider-store invocation.|

View File

@@ -0,0 +1,10 @@
# Excititor Connectors Cisco CSAF Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0298-M | DONE | Revalidated 2026-01-07. |
| AUDIT-0298-T | DONE | Revalidated 2026-01-07. |
| AUDIT-0298-A | TODO | Revalidated 2026-01-07 (open findings; pending approval). |

View File

@@ -0,0 +1,34 @@
# AGENTS
## Role
Connector for Microsoft Security Response Center (MSRC) CSAF advisories, handling authenticated downloads, throttling, and raw document persistence.
## Scope
- MSRC API onboarding (AAD client credentials), metadata discovery, and CSAF listing retrieval.
- Download pipeline with retry/backoff, checksum validation, and document deduplication.
- Mapping MSRC-specific identifiers (CVE, ADV, KB) and remediation guidance into connector metadata.
- Emitting trust metadata (AAD issuer, signing certificates) for policy weighting.
## Participants
- Worker schedules MSRC pulls honoring rate limits; WebService may trigger manual runs for urgent updates.
- CSAF normalizer processes retrieved documents into claims.
- Policy subsystem references connector trust hints for consensus scoring.
## Interfaces & contracts
- Implements `IVexConnector`, requires configuration options for tenant/client/secret or managed identity.
- Uses shared HTTP helpers, resume markers, and telemetry from Abstractions module.
## In/Out of scope
In: authenticated fetching, raw document storage, metadata mapping, retry logic.
Out: normalization/export, attestation, storage implementations (handled elsewhere).
## Observability & security expectations
- Log request batches, rate-limit responses, and token refresh events without leaking secrets.
- Track metrics for documents fetched, retries, and failure categories.
## Tests
- Connector tests with mocked MSRC endpoints and AAD token flow will live in `../StellaOps.Excititor.Connectors.MSRC.CSAF.Tests`.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,186 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
public interface IMsrcTokenProvider
{
ValueTask<MsrcAccessToken> GetAccessTokenAsync(CancellationToken cancellationToken);
}
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
{
private const string CachePrefix = "StellaOps.Excititor.Connectors.MSRC.CSAF.Token";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<MsrcTokenProvider> _logger;
private readonly TimeProvider _timeProvider;
private readonly MsrcConnectorOptions _options;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
public MsrcTokenProvider(
IHttpClientFactory httpClientFactory,
IMemoryCache cache,
IFileSystem fileSystem,
IOptions<MsrcConnectorOptions> options,
ILogger<MsrcTokenProvider> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate(_fileSystem);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async ValueTask<MsrcAccessToken> GetAccessTokenAsync(CancellationToken cancellationToken)
{
if (_options.PreferOfflineToken)
{
return LoadOfflineToken();
}
var cacheKey = CreateCacheKey();
if (_cache.TryGetValue<MsrcAccessToken>(cacheKey, out var cachedToken) &&
cachedToken is not null &&
!cachedToken.IsExpired(_timeProvider.GetUtcNow()))
{
return cachedToken;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_cache.TryGetValue<MsrcAccessToken>(cacheKey, out cachedToken) &&
cachedToken is not null &&
!cachedToken.IsExpired(_timeProvider.GetUtcNow()))
{
return cachedToken;
}
var token = await RequestTokenAsync(cancellationToken).ConfigureAwait(false);
var absoluteExpiration = token.ExpiresAt == DateTimeOffset.MaxValue
? (DateTimeOffset?)null
: token.ExpiresAt;
var options = new MemoryCacheEntryOptions();
if (absoluteExpiration.HasValue)
{
options.AbsoluteExpiration = absoluteExpiration.Value;
}
_cache.Set(cacheKey, token, options);
return token;
}
finally
{
_refreshLock.Release();
}
}
private MsrcAccessToken LoadOfflineToken()
{
if (!string.IsNullOrWhiteSpace(_options.StaticAccessToken))
{
return new MsrcAccessToken(_options.StaticAccessToken!, "Bearer", DateTimeOffset.MaxValue);
}
if (string.IsNullOrWhiteSpace(_options.OfflineTokenPath))
{
throw new InvalidOperationException("Offline token mode is enabled but no token was provided.");
}
if (!_fileSystem.File.Exists(_options.OfflineTokenPath))
{
throw new InvalidOperationException($"Offline token path '{_options.OfflineTokenPath}' does not exist.");
}
var token = _fileSystem.File.ReadAllText(_options.OfflineTokenPath).Trim();
if (string.IsNullOrEmpty(token))
{
throw new InvalidOperationException("Offline token file was empty.");
}
return new MsrcAccessToken(token, "Bearer", DateTimeOffset.MaxValue);
}
private async Task<MsrcAccessToken> RequestTokenAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Fetching MSRC AAD access token for tenant {TenantId}.", _options.TenantId);
var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.TokenClientName);
using var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
{
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["client_id"] = _options.ClientId,
["client_secret"] = _options.ClientSecret!,
["grant_type"] = "client_credentials",
["scope"] = string.IsNullOrWhiteSpace(_options.Scope) ? MsrcConnectorOptions.DefaultScope : _options.Scope,
}),
};
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException($"Failed to acquire MSRC access token ({(int)response.StatusCode}). Response: {payload}");
}
var tokenResponse = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("Token endpoint returned an empty payload.");
if (string.IsNullOrWhiteSpace(tokenResponse.AccessToken))
{
throw new InvalidOperationException("Token endpoint response did not include an access_token.");
}
var now = _timeProvider.GetUtcNow();
var expiresAt = tokenResponse.ExpiresIn > _options.ExpiryLeewaySeconds
? now.AddSeconds(tokenResponse.ExpiresIn - _options.ExpiryLeewaySeconds)
: now.AddMinutes(5);
return new MsrcAccessToken(tokenResponse.AccessToken!, tokenResponse.TokenType ?? "Bearer", expiresAt);
}
private string CreateCacheKey()
=> $"{CachePrefix}:{_options.TenantId}:{_options.ClientId}:{_options.Scope}";
private Uri BuildTokenUri()
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
public void Dispose() => _refreshLock.Dispose();
private sealed record TokenResponse
{
[JsonPropertyName("access_token")]
public string? AccessToken { get; init; }
[JsonPropertyName("token_type")]
public string? TokenType { get; init; }
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; init; }
}
}
public sealed record MsrcAccessToken(string Value, string Type, DateTimeOffset ExpiresAt)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}

View File

@@ -0,0 +1,211 @@
using System;
using System.Globalization;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
public sealed class MsrcConnectorOptions
{
public const string TokenClientName = "excititor.connector.msrc.token";
public const string DefaultScope = "https://api.msrc.microsoft.com/.default";
public const string ApiClientName = "excititor.connector.msrc.api";
public const string DefaultBaseUri = "https://api.msrc.microsoft.com/sug/v2.0/";
public const string DefaultLocale = "en-US";
public const string DefaultApiVersion = "2024-08-01";
/// <summary>
/// Azure AD tenant identifier (GUID or domain).
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application (client) identifier.
/// </summary>
public string ClientId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application secret for client credential flow.
/// </summary>
public string? ClientSecret { get; set; }
/// <summary>
/// OAuth scope requested for MSRC API access.
/// </summary>
public string Scope { get; set; } = DefaultScope;
/// <summary>
/// When true, token acquisition is skipped and the connector expects offline handling.
/// </summary>
public bool PreferOfflineToken { get; set; }
/// <summary>
/// Optional path to a pre-provisioned bearer token used when <see cref="PreferOfflineToken"/> is enabled.
/// </summary>
public string? OfflineTokenPath { get; set; }
/// <summary>
/// Optional fixed bearer token for constrained environments (e.g., short-lived offline bundles).
/// </summary>
public string? StaticAccessToken { get; set; }
/// <summary>
/// Minimum buffer (seconds) subtracted from token expiry before refresh.
/// </summary>
public int ExpiryLeewaySeconds { get; set; } = 60;
/// <summary>
/// Base URI for MSRC Security Update Guide API.
/// </summary>
public Uri BaseUri { get; set; } = new(DefaultBaseUri, UriKind.Absolute);
/// <summary>
/// Locale requested when fetching summaries.
/// </summary>
public string Locale { get; set; } = DefaultLocale;
/// <summary>
/// API version appended to MSRC requests.
/// </summary>
public string ApiVersion { get; set; } = DefaultApiVersion;
/// <summary>
/// Page size used while enumerating summaries.
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// Maximum CSAF advisories fetched per connector run.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 200;
/// <summary>
/// Overlap window applied when resuming from the last modified cursor.
/// </summary>
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
/// <summary>
/// Delay between CSAF downloads to respect rate limits.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Maximum retry attempts for summary/detail fetch operations.
/// </summary>
public int MaxRetryAttempts { get; set; } = 3;
/// <summary>
/// Base delay applied between retries (jitter handled by connector).
/// </summary>
public TimeSpan RetryBaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Optional lower bound for initial synchronisation when no cursor is stored.
/// </summary>
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
/// <summary>
/// Maximum number of document digests persisted for deduplication.
/// </summary>
public int MaxTrackedDigests { get; set; } = 2048;
public void Validate(IFileSystem? fileSystem = null)
{
if (PreferOfflineToken)
{
if (string.IsNullOrWhiteSpace(OfflineTokenPath) && string.IsNullOrWhiteSpace(StaticAccessToken))
{
throw new InvalidOperationException("OfflineTokenPath or StaticAccessToken must be provided when PreferOfflineToken is enabled.");
}
}
else
{
if (string.IsNullOrWhiteSpace(TenantId))
{
throw new InvalidOperationException("TenantId is required when not operating in offline token mode.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("ClientId is required when not operating in offline token mode.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("ClientSecret is required when not operating in offline token mode.");
}
}
if (string.IsNullOrWhiteSpace(Scope))
{
Scope = DefaultScope;
}
if (ExpiryLeewaySeconds < 10)
{
ExpiryLeewaySeconds = 10;
}
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("BaseUri must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(Locale))
{
throw new InvalidOperationException("Locale must be provided.");
}
if (!CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
}
if (string.IsNullOrWhiteSpace(ApiVersion))
{
throw new InvalidOperationException("ApiVersion must be provided.");
}
if (PageSize <= 0 || PageSize > 500)
{
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
{
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
}
if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10))
{
throw new InvalidOperationException($"{nameof(RequestDelay)} must be between 0 and 10 seconds.");
}
if (MaxRetryAttempts <= 0 || MaxRetryAttempts > 10)
{
throw new InvalidOperationException($"{nameof(MaxRetryAttempts)} must be between 1 and 10.");
}
if (RetryBaseDelay < TimeSpan.Zero || RetryBaseDelay > TimeSpan.FromMinutes(5))
{
throw new InvalidOperationException($"{nameof(RetryBaseDelay)} must be between 0 and 5 minutes.");
}
if (MaxTrackedDigests <= 0 || MaxTrackedDigests > 10000)
{
throw new InvalidOperationException($"{nameof(MaxTrackedDigests)} must be between 1 and 10000.");
}
if (!string.IsNullOrWhiteSpace(OfflineTokenPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineTokenPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,59 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using StellaOps.Excititor.Core;
using System;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.DependencyInjection;
public static class MsrcConnectorServiceCollectionExtensions
{
public static IServiceCollection AddMsrcCsafConnector(this IServiceCollection services, Action<MsrcConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<MsrcConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddHttpClient(MsrcConnectorOptions.TokenClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddHttpClient(MsrcConnectorOptions.ApiClientName)
.ConfigureHttpClient((provider, client) =>
{
var options = provider.GetRequiredService<IOptions<MsrcConnectorOptions>>().Value;
client.BaseAddress = options.BaseUri;
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<IMsrcTokenProvider, MsrcTokenProvider>();
services.AddSingleton<IVexConnector, MsrcCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,590 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions.Trust;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF;
public sealed class MsrcCsafConnector : VexConnectorBase
{
private const string QuarantineMetadataKey = "excititor.quarantine.reason";
private const string FormatMetadataKey = "msrc.csaf.format";
private const string VulnerabilityMetadataKey = "msrc.vulnerabilityId";
private const string AdvisoryIdMetadataKey = "msrc.advisoryId";
private const string LastModifiedMetadataKey = "msrc.lastModified";
private const string ReleaseDateMetadataKey = "msrc.releaseDate";
private const string CvssSeverityMetadataKey = "msrc.severity";
private const string CvrfUrlMetadataKey = "msrc.cvrfUrl";
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:msrc",
kind: VexProviderKind.Vendor,
displayName: "Microsoft MSRC CSAF")
{
Description = "Authenticated connector for Microsoft Security Response Center CSAF advisories.",
SupportedFormats = ImmutableArray.Create(VexDocumentFormat.Csaf),
Tags = ImmutableArray.Create("microsoft", "csaf", "vendor"),
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMsrcTokenProvider _tokenProvider;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IOptions<MsrcConnectorOptions> _options;
private readonly ILogger<MsrcCsafConnector> _logger;
private readonly Func<double> _jitterSource;
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
private MsrcConnectorOptions? _validatedOptions;
public MsrcCsafConnector(
IHttpClientFactory httpClientFactory,
IMsrcTokenProvider tokenProvider,
IVexConnectorStateRepository stateRepository,
IOptions<MsrcConnectorOptions> options,
ILogger<MsrcCsafConnector> logger,
TimeProvider timeProvider,
Func<double>? jitterSource = null)
: base(DescriptorInstance, logger, timeProvider)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_jitterSource = jitterSource ?? Random.Shared.NextDouble;
}
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered.");
options.Validate();
_validatedOptions = options;
LogConnectorEvent(
LogLevel.Information,
"validate",
"Validated MSRC CSAF connector options.",
new Dictionary<string, object?>
{
["baseUri"] = options.BaseUri.ToString(),
["locale"] = options.Locale,
["apiVersion"] = options.ApiVersion,
["pageSize"] = options.PageSize,
["maxAdvisories"] = options.MaxAdvisoriesPerFetch,
});
return ValueTask.CompletedTask;
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(
VexConnectorContext context,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var options = EnsureOptionsValidated();
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var (from, to) = CalculateWindow(context.Since, state, options);
LogConnectorEvent(
LogLevel.Information,
"fetch.window",
$"Fetching MSRC CSAF advisories updated between {from:O} and {to:O}.",
new Dictionary<string, object?>
{
["from"] = from,
["to"] = to,
["cursorOverlapSeconds"] = options.CursorOverlap.TotalSeconds,
});
var client = await CreateAuthenticatedClientAsync(options, cancellationToken).ConfigureAwait(false);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var latest = state?.LastUpdated ?? from;
var fetched = 0;
var stateChanged = false;
await foreach (var summary in EnumerateSummariesAsync(client, options, from, to, cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
if (fetched >= options.MaxAdvisoriesPerFetch)
{
break;
}
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
LogConnectorEvent(LogLevel.Debug, "skip.no-cvrf", $"Skipping MSRC advisory {summary.Id} because no CSAF URL was provided.");
continue;
}
var documentUri = ResolveCvrfUri(options.BaseUri, summary.CvrfUrl);
VexRawDocument? rawDocument = null;
try
{
rawDocument = await DownloadCsafAsync(client, summary, documentUri, options, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogConnectorEvent(LogLevel.Warning, "fetch.error", $"Failed to download MSRC CSAF package {documentUri}.", new Dictionary<string, object?>
{
["advisoryId"] = summary.Id,
["vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
}, ex);
await Task.Delay(GetRetryDelay(options, 1), cancellationToken).ConfigureAwait(false);
continue;
}
if (!digestSet.Add(rawDocument.Digest))
{
LogConnectorEvent(LogLevel.Debug, "skip.duplicate", $"Skipping MSRC CSAF package {documentUri} because it was already processed.");
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
fetched++;
latest = DetermineLatest(summary, latest) ?? latest;
var quarantineReason = rawDocument.Metadata.TryGetValue(QuarantineMetadataKey, out var reason) ? reason : null;
if (quarantineReason is not null)
{
LogConnectorEvent(LogLevel.Warning, "quarantine", $"Quarantined MSRC CSAF package {documentUri} ({quarantineReason}).");
continue;
}
yield return rawDocument;
if (options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
if (stateChanged)
{
if (digestList.Count > options.MaxTrackedDigests)
{
var trimmed = digestList.Count - options.MaxTrackedDigests;
digestList.RemoveRange(0, trimmed);
}
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latest == DateTimeOffset.MinValue ? state?.LastUpdated : latest,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
LogConnectorEvent(
LogLevel.Information,
"fetch.completed",
$"MSRC CSAF fetch completed with {fetched} new documents.",
new Dictionary<string, object?>
{
["fetched"] = fetched,
["stateChanged"] = stateChanged,
["lastUpdated"] = latest,
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("MSRC CSAF connector relies on CSAF normalizers for document processing.");
private async Task<VexRawDocument> DownloadCsafAsync(
HttpClient client,
MsrcVulnerabilitySummary summary,
Uri documentUri,
MsrcConnectorOptions options,
CancellationToken cancellationToken)
{
using var response = await SendWithRetryAsync(
client,
() => new HttpRequestMessage(HttpMethod.Get, documentUri),
options,
cancellationToken).ConfigureAwait(false);
var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var validation = ValidateCsafPayload(payload);
var metadata = BuildMetadata(builder =>
{
builder.Add(AdvisoryIdMetadataKey, summary.Id);
builder.Add(VulnerabilityMetadataKey, summary.VulnerabilityId ?? summary.Id);
builder.Add(CvrfUrlMetadataKey, documentUri.ToString());
builder.Add(FormatMetadataKey, validation.Format);
if (!string.IsNullOrWhiteSpace(summary.Severity))
{
builder.Add(CvssSeverityMetadataKey, summary.Severity);
}
if (summary.LastModifiedDate is not null)
{
builder.Add(LastModifiedMetadataKey, summary.LastModifiedDate.Value.ToString("O", CultureInfo.InvariantCulture));
}
if (summary.ReleaseDate is not null)
{
builder.Add(ReleaseDateMetadataKey, summary.ReleaseDate.Value.ToString("O", CultureInfo.InvariantCulture));
}
if (!string.IsNullOrWhiteSpace(validation.QuarantineReason))
{
builder.Add(QuarantineMetadataKey, validation.QuarantineReason);
}
if (response.Headers.ETag is not null)
{
builder.Add("http.etag", response.Headers.ETag.Tag);
}
if (response.Content.Headers.LastModified is { } lastModified)
{
builder.Add("http.lastModified", lastModified.ToString("O", CultureInfo.InvariantCulture));
}
ConnectorSignerMetadataEnricher.Enrich(builder, Descriptor.Id, _logger);
});
return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, payload, metadata);
}
private async Task<HttpClient> CreateAuthenticatedClientAsync(MsrcConnectorOptions options, CancellationToken cancellationToken)
{
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.ApiClientName);
client.DefaultRequestHeaders.Remove("Authorization");
client.DefaultRequestHeaders.Add("Authorization", $"{token.Type} {token.Value}");
client.DefaultRequestHeaders.Remove("Accept-Language");
client.DefaultRequestHeaders.Add("Accept-Language", options.Locale);
client.DefaultRequestHeaders.Remove("api-version");
client.DefaultRequestHeaders.Add("api-version", options.ApiVersion);
client.DefaultRequestHeaders.Remove("Accept");
client.DefaultRequestHeaders.Add("Accept", "application/json");
return client;
}
private async Task<HttpResponseMessage> SendWithRetryAsync(
HttpClient client,
Func<HttpRequestMessage> requestFactory,
MsrcConnectorOptions options,
CancellationToken cancellationToken)
{
Exception? lastError = null;
HttpResponseMessage? response = null;
for (var attempt = 1; attempt <= options.MaxRetryAttempts; attempt++)
{
response?.Dispose();
using var request = requestFactory();
try
{
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return response;
}
if (!ShouldRetry(response.StatusCode) || attempt == options.MaxRetryAttempts)
{
response.EnsureSuccessStatusCode();
}
}
catch (Exception ex) when (IsTransient(ex) && attempt < options.MaxRetryAttempts)
{
lastError = ex;
LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex);
}
catch (Exception)
{
response?.Dispose();
throw;
}
await Task.Delay(GetRetryDelay(options, attempt), cancellationToken).ConfigureAwait(false);
}
response?.Dispose();
throw lastError ?? new InvalidOperationException("MSRC request retries exhausted.");
}
private TimeSpan GetRetryDelay(MsrcConnectorOptions options, int attempt)
{
var baseDelay = options.RetryBaseDelay.TotalMilliseconds;
var multiplier = Math.Pow(2, Math.Max(0, attempt - 1));
var jitter = _jitterSource() * baseDelay * 0.25;
var delayMs = Math.Min(baseDelay * multiplier + jitter, TimeSpan.FromMinutes(5).TotalMilliseconds);
return TimeSpan.FromMilliseconds(delayMs);
}
private async IAsyncEnumerable<MsrcVulnerabilitySummary> EnumerateSummariesAsync(
HttpClient client,
MsrcConnectorOptions options,
DateTimeOffset from,
DateTimeOffset to,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var fetched = 0;
var requestUri = BuildSummaryUri(options, from, to);
while (requestUri is not null && fetched < options.MaxAdvisoriesPerFetch)
{
using var response = await SendWithRetryAsync(
client,
() => new HttpRequestMessage(HttpMethod.Get, requestUri),
options,
cancellationToken).ConfigureAwait(false);
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var payload = await JsonSerializer.DeserializeAsync<MsrcSummaryResponse>(stream, _serializerOptions, cancellationToken).ConfigureAwait(false)
?? new MsrcSummaryResponse();
foreach (var summary in payload.Value)
{
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
continue;
}
yield return summary;
fetched++;
if (fetched >= options.MaxAdvisoriesPerFetch)
{
yield break;
}
}
if (string.IsNullOrWhiteSpace(payload.NextLink))
{
break;
}
if (!Uri.TryCreate(payload.NextLink, UriKind.Absolute, out requestUri))
{
LogConnectorEvent(LogLevel.Warning, "pagination.invalid", $"MSRC pagination returned invalid next link '{payload.NextLink}'.");
break;
}
}
}
private static Uri BuildSummaryUri(MsrcConnectorOptions options, DateTimeOffset from, DateTimeOffset to)
{
var baseText = options.BaseUri.ToString().TrimEnd('/');
var builder = new StringBuilder(baseText.Length + 128);
builder.Append(baseText);
if (!baseText.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase))
{
builder.Append("/vulnerabilities");
}
builder.Append("?");
builder.Append("$top=").Append(options.PageSize);
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(from.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)));
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(to.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)));
builder.Append("&$orderby=lastModifiedDate");
builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale));
builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion));
return new Uri(builder.ToString(), UriKind.Absolute);
}
private (DateTimeOffset From, DateTimeOffset To) CalculateWindow(
DateTimeOffset? contextSince,
VexConnectorState? state,
MsrcConnectorOptions options)
{
var now = UtcNow();
var since = contextSince ?? state?.LastUpdated ?? options.InitialLastModified ?? now.AddDays(-30);
if (state?.LastUpdated is { } persisted && persisted > since)
{
since = persisted;
}
if (options.CursorOverlap > TimeSpan.Zero)
{
since = since.Add(-options.CursorOverlap);
}
if (since < now.AddYears(-20))
{
since = now.AddYears(-20);
}
return (since, now);
}
private static bool ShouldRetry(HttpStatusCode statusCode)
=> statusCode == HttpStatusCode.TooManyRequests ||
(int)statusCode >= 500;
private static bool IsTransient(Exception exception)
=> exception is HttpRequestException or IOException or TaskCanceledException;
private static Uri ResolveCvrfUri(Uri baseUri, string cvrfUrl)
=> Uri.TryCreate(cvrfUrl, UriKind.Absolute, out var absolute)
? absolute
: new Uri(baseUri, cvrfUrl);
private static CsafValidationResult ValidateCsafPayload(ReadOnlyMemory<byte> payload)
{
try
{
if (IsZip(payload.Span))
{
using var zipStream = new MemoryStream(payload.ToArray(), writable: false);
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true);
var entry = archive.Entries.FirstOrDefault(e => e.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
?? archive.Entries.FirstOrDefault();
if (entry is null)
{
return new CsafValidationResult("zip", "Zip archive did not contain any entries.");
}
using var entryStream = entry.Open();
using var reader = new StreamReader(entryStream, Encoding.UTF8);
using var json = JsonDocument.Parse(reader.ReadToEnd());
return CsafValidationResult.Valid("zip");
}
if (IsGzip(payload.Span))
{
using var input = new MemoryStream(payload.ToArray(), writable: false);
using var gzip = new GZipStream(input, CompressionMode.Decompress);
using var reader = new StreamReader(gzip, Encoding.UTF8);
using var json = JsonDocument.Parse(reader.ReadToEnd());
return CsafValidationResult.Valid("gzip");
}
using var jsonDocument = JsonDocument.Parse(payload);
return CsafValidationResult.Valid("json");
}
catch (JsonException ex)
{
var failedFormat = IsZip(payload.Span) ? "zip" : IsGzip(payload.Span) ? "gzip" : "json";
return new CsafValidationResult(failedFormat, $"JSON parse failed: {ex.Message}");
}
catch (InvalidDataException ex)
{
return new CsafValidationResult("invalid", ex.Message);
}
catch (EndOfStreamException ex)
{
return new CsafValidationResult("invalid", ex.Message);
}
}
private static bool IsZip(ReadOnlySpan<byte> content)
=> content.Length > 3 && content[0] == 0x50 && content[1] == 0x4B;
private static bool IsGzip(ReadOnlySpan<byte> content)
=> content.Length > 2 && content[0] == 0x1F && content[1] == 0x8B;
private static DateTimeOffset? DetermineLatest(MsrcVulnerabilitySummary summary, DateTimeOffset? current)
{
var candidate = summary.LastModifiedDate ?? summary.ReleaseDate;
if (candidate is null)
{
return current;
}
if (current is null || candidate > current)
{
return candidate;
}
return current;
}
private MsrcConnectorOptions EnsureOptionsValidated()
{
if (_validatedOptions is not null)
{
return _validatedOptions;
}
var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered.");
options.Validate();
_validatedOptions = options;
return options;
}
private sealed record CsafValidationResult(string Format, string? QuarantineReason)
{
public static CsafValidationResult Valid(string format) => new(format, null);
}
}
internal sealed record MsrcSummaryResponse
{
[JsonPropertyName("value")]
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
[JsonPropertyName("@odata.nextLink")]
public string? NextLink { get; init; }
}
internal sealed record MsrcVulnerabilitySummary
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string? VulnerabilityId { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}

View File

@@ -0,0 +1,6 @@
using StellaOps.Plugin.Versioning;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Connectors.MSRC.CSAF.Tests")]
[assembly: StellaPluginVersion("1.0.0", MinimumHostVersion = "1.0.0", MaximumHostVersion = "1.99.99")]

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Persistence\StellaOps.Excititor.Persistence.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="System.IO.Abstractions" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,6 @@
# Completed Tasks
|EXCITITOR-CONN-MS-01-001 AAD onboarding & token cache|Team Excititor Connectors MSRC|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added MSRC connector project with configurable AAD options, token provider (offline/online modes), DI wiring, and unit tests covering caching and fallback scenarios.|
|EXCITITOR-CONN-MS-01-002 CSAF download pipeline|Team Excititor Connectors MSRC|EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-29)** Implemented authenticated CSAF retrieval with retry/backoff, checksum enforcement, quarantine for invalid archives, and regression tests covering dedupe + idempotent state updates.|

View File

@@ -0,0 +1,10 @@
# Excititor Connectors MSRC CSAF Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0300-M | DONE | Revalidated 2026-01-07. |
| AUDIT-0300-T | DONE | Revalidated 2026-01-07. |
| AUDIT-0300-A | TODO | Revalidated 2026-01-07 (open findings; pending approval). |

View File

@@ -0,0 +1,34 @@
# AGENTS
## Role
Connector for OCI registry OpenVEX attestations, discovering images, downloading attestations, and projecting statements into raw storage.
## Scope
- OCI registry discovery, authentication (cosign OIDC/key), and ref resolution for provided image digests/tags.
- Fetching DSSE envelopes, verifying signatures (delegated to Attestation module), and persisting raw statements.
- Mapping OCI manifest metadata (repository, digest, subject) to connector provenance.
- Managing offline bundles that seed attestations without registry access.
## Participants
- Worker schedules polls for configured registries/images; WebService supports manual refresh.
- OpenVEX normalizer consumes statements to create claims.
- Attestation module is reused to verify upstream envelopes prior to storage.
## Interfaces & contracts
- Implements `IVexConnector` with options for image list, auth, parallelism, and offline file seeds.
- Utilizes shared abstractions for retries, telemetry, and resume markers.
## In/Out of scope
In: OCI interaction, attestation retrieval, verification trigger, raw persistence.
Out: normalization/export, policy evaluation, storage implementation.
## Observability & security expectations
- Log image references, attestation counts, verification outcomes; redact credentials.
- Emit metrics for attestation reuse ratio, verification duration, and failures.
## Tests
- Connector tests with mock OCI registry/attestation responses will live in `../StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests`.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,111 @@
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using System;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
public sealed record CosignKeylessIdentity(
string Issuer,
string Subject,
Uri? FulcioUrl,
Uri? RekorUrl,
string? ClientId,
string? ClientSecret,
string? Audience,
string? IdentityToken);
public sealed record CosignKeyPairIdentity(
string PrivateKeyPath,
string? Password,
string? CertificatePath,
Uri? RekorUrl,
string? FulcioRootPath);
public sealed record OciCosignAuthority(
CosignCredentialMode Mode,
CosignKeylessIdentity? Keyless,
CosignKeyPairIdentity? KeyPair,
bool RequireSignature,
TimeSpan VerifyTimeout);
public static class OciCosignAuthorityFactory
{
public static OciCosignAuthority Create(OciCosignVerificationOptions options, IFileSystem? fileSystem = null)
{
ArgumentNullException.ThrowIfNull(options);
CosignKeylessIdentity? keyless = null;
CosignKeyPairIdentity? keyPair = null;
switch (options.Mode)
{
case CosignCredentialMode.None:
break;
case CosignCredentialMode.Keyless:
keyless = CreateKeyless(options.Keyless);
break;
case CosignCredentialMode.KeyPair:
keyPair = CreateKeyPair(options.KeyPair, fileSystem);
break;
default:
throw new InvalidOperationException($"Unsupported Cosign credential mode '{options.Mode}'.");
}
return new OciCosignAuthority(
Mode: options.Mode,
Keyless: keyless,
KeyPair: keyPair,
RequireSignature: options.RequireSignature,
VerifyTimeout: options.VerifyTimeout);
}
private static CosignKeylessIdentity CreateKeyless(CosignKeylessOptions options)
{
ArgumentNullException.ThrowIfNull(options);
Uri? fulcio = null;
Uri? rekor = null;
if (!string.IsNullOrWhiteSpace(options.FulcioUrl))
{
fulcio = new Uri(options.FulcioUrl, UriKind.Absolute);
}
if (!string.IsNullOrWhiteSpace(options.RekorUrl))
{
rekor = new Uri(options.RekorUrl, UriKind.Absolute);
}
return new CosignKeylessIdentity(
Issuer: options.Issuer!,
Subject: options.Subject!,
FulcioUrl: fulcio,
RekorUrl: rekor,
ClientId: options.ClientId,
ClientSecret: options.ClientSecret,
Audience: options.Audience,
IdentityToken: options.IdentityToken);
}
private static CosignKeyPairIdentity CreateKeyPair(CosignKeyPairOptions options, IFileSystem? fileSystem)
{
ArgumentNullException.ThrowIfNull(options);
Uri? rekor = null;
if (!string.IsNullOrWhiteSpace(options.RekorUrl))
{
rekor = new Uri(options.RekorUrl, UriKind.Absolute);
}
return new CosignKeyPairIdentity(
PrivateKeyPath: options.PrivateKeyPath!,
Password: options.Password,
CertificatePath: options.CertificatePath,
RekorUrl: rekor,
FulcioRootPath: options.FulcioRootPath);
}
}

View File

@@ -0,0 +1,60 @@
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using System;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
public enum OciRegistryAuthMode
{
Anonymous = 0,
Basic = 1,
IdentityToken = 2,
RefreshToken = 3,
}
public sealed record OciRegistryAuthorization(
string? RegistryAuthority,
OciRegistryAuthMode Mode,
string? Username,
string? Password,
string? IdentityToken,
string? RefreshToken,
bool AllowAnonymousFallback)
{
public static OciRegistryAuthorization Create(OciRegistryAuthenticationOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var mode = OciRegistryAuthMode.Anonymous;
string? username = null;
string? password = null;
string? identityToken = null;
string? refreshToken = null;
if (!string.IsNullOrWhiteSpace(options.IdentityToken))
{
mode = OciRegistryAuthMode.IdentityToken;
identityToken = options.IdentityToken;
}
else if (!string.IsNullOrWhiteSpace(options.RefreshToken))
{
mode = OciRegistryAuthMode.RefreshToken;
refreshToken = options.RefreshToken;
}
else if (!string.IsNullOrWhiteSpace(options.Username))
{
mode = OciRegistryAuthMode.Basic;
username = options.Username;
password = options.Password;
}
return new OciRegistryAuthorization(
RegistryAuthority: options.RegistryAuthority,
Mode: mode,
Username: username,
Password: password,
IdentityToken: identityToken,
RefreshToken: refreshToken,
AllowAnonymousFallback: options.AllowAnonymousFallback);
}
}

View File

@@ -0,0 +1,322 @@
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
public sealed class OciOpenVexAttestationConnectorOptions
{
public const string HttpClientName = "excititor.connector.oci.openvex.attest";
public IList<OciImageSubscriptionOptions> Images { get; } = new List<OciImageSubscriptionOptions>();
public OciRegistryAuthenticationOptions Registry { get; } = new();
public OciCosignVerificationOptions Cosign { get; } = new();
public OciOfflineBundleOptions Offline { get; } = new();
public TimeSpan DiscoveryCacheDuration { get; set; } = TimeSpan.FromMinutes(15);
public int MaxParallelResolutions { get; set; } = 4;
public bool AllowHttpRegistries { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (Images.Count == 0)
{
throw new InvalidOperationException("At least one OCI image reference must be configured.");
}
foreach (var image in Images)
{
image.Validate();
}
if (MaxParallelResolutions <= 0 || MaxParallelResolutions > 32)
{
throw new InvalidOperationException("MaxParallelResolutions must be between 1 and 32.");
}
if (DiscoveryCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("DiscoveryCacheDuration must be a positive time span.");
}
Registry.Validate();
Cosign.Validate(fileSystem);
Offline.Validate(fileSystem);
if (!AllowHttpRegistries && Images.Any(i => i.Reference is not null && i.Reference.StartsWith("http://", StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException("HTTP (non-TLS) registries are disabled. Enable AllowHttpRegistries to permit them.");
}
}
}
public sealed class OciImageSubscriptionOptions
{
private OciImageReference? _parsedReference;
/// <summary>
/// Gets or sets the OCI reference (e.g. registry.example.com/repository:tag or registry.example.com/repository@sha256:abcdef).
/// </summary>
public string? Reference { get; set; }
/// <summary>
/// Optional friendly name used in logs when referencing this subscription.
/// </summary>
public string? DisplayName { get; set; }
/// <summary>
/// Optional file path for an offline attestation bundle associated with this image.
/// </summary>
public string? OfflineBundlePath { get; set; }
/// <summary>
/// Optional override for the expected subject digest. When provided, discovery will verify resolved digests match.
/// </summary>
public string? ExpectedSubjectDigest { get; set; }
internal OciImageReference? ParsedReference => _parsedReference;
public void Validate()
{
if (string.IsNullOrWhiteSpace(Reference))
{
throw new InvalidOperationException("Image Reference is required for OCI OpenVEX attestation connector.");
}
_parsedReference = OciImageReferenceParser.Parse(Reference);
if (!string.IsNullOrWhiteSpace(ExpectedSubjectDigest))
{
if (!ExpectedSubjectDigest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("ExpectedSubjectDigest must start with 'sha256:'.");
}
if (ExpectedSubjectDigest.Length != "sha256:".Length + 64)
{
throw new InvalidOperationException("ExpectedSubjectDigest must contain a 64-character hexadecimal hash.");
}
}
}
}
public sealed class OciRegistryAuthenticationOptions
{
/// <summary>
/// Optional registry authority filter (e.g. registry.example.com:5000). When set it must match image references.
/// </summary>
public string? RegistryAuthority { get; set; }
public string? Username { get; set; }
public string? Password { get; set; }
public string? IdentityToken { get; set; }
public string? RefreshToken { get; set; }
public bool AllowAnonymousFallback { get; set; } = true;
public void Validate()
{
var hasUser = !string.IsNullOrWhiteSpace(Username);
var hasPassword = !string.IsNullOrWhiteSpace(Password);
var hasIdentityToken = !string.IsNullOrWhiteSpace(IdentityToken);
var hasRefreshToken = !string.IsNullOrWhiteSpace(RefreshToken);
if (hasIdentityToken && (hasUser || hasPassword))
{
throw new InvalidOperationException("IdentityToken cannot be combined with Username/Password for OCI registry authentication.");
}
if (hasRefreshToken && (hasUser || hasPassword))
{
throw new InvalidOperationException("RefreshToken cannot be combined with Username/Password for OCI registry authentication.");
}
if (hasUser != hasPassword)
{
throw new InvalidOperationException("Username and Password must be provided together for OCI registry authentication.");
}
if (!string.IsNullOrWhiteSpace(RegistryAuthority) && RegistryAuthority.Contains('/', StringComparison.Ordinal))
{
throw new InvalidOperationException("RegistryAuthority must not contain path segments.");
}
}
}
public sealed class OciCosignVerificationOptions
{
public CosignCredentialMode Mode { get; set; } = CosignCredentialMode.Keyless;
public CosignKeylessOptions Keyless { get; } = new();
public CosignKeyPairOptions KeyPair { get; } = new();
public bool RequireSignature { get; set; } = true;
public TimeSpan VerifyTimeout { get; set; } = TimeSpan.FromSeconds(30);
public void Validate(IFileSystem? fileSystem = null)
{
if (VerifyTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("VerifyTimeout must be a positive time span.");
}
switch (Mode)
{
case CosignCredentialMode.None:
break;
case CosignCredentialMode.Keyless:
Keyless.Validate();
break;
case CosignCredentialMode.KeyPair:
KeyPair.Validate(fileSystem);
break;
default:
throw new InvalidOperationException($"Unsupported Cosign credential mode '{Mode}'.");
}
}
}
public enum CosignCredentialMode
{
None = 0,
Keyless = 1,
KeyPair = 2,
}
public sealed class CosignKeylessOptions
{
public string? Issuer { get; set; }
public string? Subject { get; set; }
public string? FulcioUrl { get; set; } = "https://fulcio.sigstore.dev";
public string? RekorUrl { get; set; } = "https://rekor.sigstore.dev";
public string? ClientId { get; set; }
public string? ClientSecret { get; set; }
public string? Audience { get; set; }
public string? IdentityToken { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Issuer))
{
throw new InvalidOperationException("Cosign keyless Issuer must be provided.");
}
if (string.IsNullOrWhiteSpace(Subject))
{
throw new InvalidOperationException("Cosign keyless Subject must be provided.");
}
if (!string.IsNullOrWhiteSpace(FulcioUrl) && !Uri.TryCreate(FulcioUrl, UriKind.Absolute, out var fulcio))
{
throw new InvalidOperationException("FulcioUrl must be an absolute URI when provided.");
}
if (!string.IsNullOrWhiteSpace(RekorUrl) && !Uri.TryCreate(RekorUrl, UriKind.Absolute, out var rekor))
{
throw new InvalidOperationException("RekorUrl must be an absolute URI when provided.");
}
if (!string.IsNullOrWhiteSpace(ClientSecret) && string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("Cosign keyless ClientId must be provided when ClientSecret is specified.");
}
}
}
public sealed class CosignKeyPairOptions
{
public string? PrivateKeyPath { get; set; }
public string? Password { get; set; }
public string? CertificatePath { get; set; }
public string? RekorUrl { get; set; }
public string? FulcioRootPath { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (string.IsNullOrWhiteSpace(PrivateKeyPath))
{
throw new InvalidOperationException("PrivateKeyPath must be provided for Cosign key pair mode.");
}
var fs = fileSystem ?? new FileSystem();
if (!fs.File.Exists(PrivateKeyPath))
{
throw new InvalidOperationException($"Cosign private key file not found: {PrivateKeyPath}");
}
if (!string.IsNullOrWhiteSpace(CertificatePath) && !fs.File.Exists(CertificatePath))
{
throw new InvalidOperationException($"Cosign certificate file not found: {CertificatePath}");
}
if (!string.IsNullOrWhiteSpace(FulcioRootPath) && !fs.File.Exists(FulcioRootPath))
{
throw new InvalidOperationException($"Cosign Fulcio root file not found: {FulcioRootPath}");
}
if (!string.IsNullOrWhiteSpace(RekorUrl) && !Uri.TryCreate(RekorUrl, UriKind.Absolute, out _))
{
throw new InvalidOperationException("RekorUrl must be an absolute URI when provided for Cosign key pair mode.");
}
}
}
public sealed class OciOfflineBundleOptions
{
public string? RootDirectory { get; set; }
public bool PreferOffline { get; set; }
public bool AllowNetworkFallback { get; set; } = true;
public string? DefaultBundleFileName { get; set; } = "openvex-attestations.tgz";
public bool RequireBundles { get; set; }
public void Validate(IFileSystem? fileSystem = null)
{
if (string.IsNullOrWhiteSpace(RootDirectory))
{
return;
}
var fs = fileSystem ?? new FileSystem();
if (!fs.Directory.Exists(RootDirectory))
{
if (PreferOffline || RequireBundles)
{
throw new InvalidOperationException($"Offline bundle root directory '{RootDirectory}' does not exist.");
}
fs.Directory.CreateDirectory(RootDirectory);
}
}
}

View File

@@ -0,0 +1,36 @@
using StellaOps.Excititor.Connectors.Abstractions;
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
public sealed class OciOpenVexAttestationConnectorOptionsValidator : IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public OciOpenVexAttestationConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(
VexConnectorDescriptor descriptor,
OciOpenVexAttestationConnectorOptions options,
IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,53 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
using StellaOps.Excititor.Core;
using System;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection;
public static class OciOpenVexAttestationConnectorServiceCollectionExtensions
{
public static IServiceCollection AddOciOpenVexAttestationConnector(
this IServiceCollection services,
Action<OciOpenVexAttestationConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<OciOpenVexAttestationConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
});
services.AddSingleton<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>, OciOpenVexAttestationConnectorOptionsValidator>();
services.AddSingleton<OciAttestationDiscoveryService>();
services.AddSingleton<OciAttestationFetcher>();
services.AddSingleton<IVexConnector, OciOpenVexAttestationConnector>();
services.AddHttpClient(OciOpenVexAttestationConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.cncf.openvex.v1+json");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
return services;
}
}

View File

@@ -0,0 +1,12 @@
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciAttestationDiscoveryResult(
ImmutableArray<OciAttestationTarget> Targets,
OciRegistryAuthorization RegistryAuthorization,
OciCosignAuthority CosignAuthority,
bool PreferOffline,
bool AllowNetworkFallback);

View File

@@ -0,0 +1,189 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed class OciAttestationDiscoveryService
{
private readonly IMemoryCache _memoryCache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<OciAttestationDiscoveryService> _logger;
public OciAttestationDiscoveryService(
IMemoryCache memoryCache,
IFileSystem fileSystem,
ILogger<OciAttestationDiscoveryService> logger)
{
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public Task<OciAttestationDiscoveryResult> LoadAsync(
OciOpenVexAttestationConnectorOptions options,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue(cacheKey, out OciAttestationDiscoveryResult? cached) && cached is not null)
{
_logger.LogDebug("Using cached OCI attestation discovery result for {ImageCount} images.", cached.Targets.Length);
return Task.FromResult(cached);
}
var targets = new List<OciAttestationTarget>(options.Images.Count);
foreach (var image in options.Images)
{
cancellationToken.ThrowIfCancellationRequested();
var parsed = image.ParsedReference ?? OciImageReferenceParser.Parse(image.Reference!);
var offlinePath = ResolveOfflinePath(options, image, parsed);
OciOfflineBundleReference? offline = null;
if (!string.IsNullOrWhiteSpace(offlinePath))
{
var fullPath = _fileSystem.Path.GetFullPath(offlinePath!);
var exists = _fileSystem.File.Exists(fullPath) || _fileSystem.Directory.Exists(fullPath);
if (!exists && options.Offline.RequireBundles)
{
throw new InvalidOperationException($"Required offline bundle '{fullPath}' for reference '{parsed.Canonical}' was not found.");
}
offline = new OciOfflineBundleReference(fullPath, exists, image.ExpectedSubjectDigest);
}
targets.Add(new OciAttestationTarget(parsed, image.ExpectedSubjectDigest, offline));
}
var authorization = OciRegistryAuthorization.Create(options.Registry);
var cosignAuthority = OciCosignAuthorityFactory.Create(options.Cosign, _fileSystem);
var result = new OciAttestationDiscoveryResult(
targets.ToImmutableArray(),
authorization,
cosignAuthority,
options.Offline.PreferOffline,
options.Offline.AllowNetworkFallback);
_memoryCache.Set(cacheKey, result, options.DiscoveryCacheDuration);
return Task.FromResult(result);
}
private string? ResolveOfflinePath(
OciOpenVexAttestationConnectorOptions options,
OciImageSubscriptionOptions image,
OciImageReference parsed)
{
if (!string.IsNullOrWhiteSpace(image.OfflineBundlePath))
{
return image.OfflineBundlePath;
}
if (string.IsNullOrWhiteSpace(options.Offline.RootDirectory))
{
return null;
}
var root = options.Offline.RootDirectory!;
var segments = new List<string> { SanitizeSegment(parsed.Registry) };
var repositoryParts = parsed.Repository.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (repositoryParts.Length == 0)
{
segments.Add(SanitizeSegment(parsed.Repository));
}
else
{
foreach (var part in repositoryParts)
{
segments.Add(SanitizeSegment(part));
}
}
var versionSegment = parsed.Digest is not null
? SanitizeSegment(parsed.Digest)
: SanitizeSegment(parsed.Tag ?? "latest");
segments.Add(versionSegment);
var combined = _fileSystem.Path.Combine(new[] { root }.Concat(segments).ToArray());
if (!string.IsNullOrWhiteSpace(options.Offline.DefaultBundleFileName))
{
combined = _fileSystem.Path.Combine(combined, options.Offline.DefaultBundleFileName!);
}
return combined;
}
private static string SanitizeSegment(string value)
{
if (string.IsNullOrEmpty(value))
{
return "_";
}
var builder = new StringBuilder(value.Length);
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch) || ch is '-' or '_' or '.')
{
builder.Append(ch);
}
else
{
builder.Append('_');
}
}
return builder.Length == 0 ? "_" : builder.ToString();
}
private static string CreateCacheKey(OciOpenVexAttestationConnectorOptions options)
{
using var sha = SHA256.Create();
var builder = new StringBuilder();
builder.AppendLine("oci-openvex-attest");
builder.AppendLine(options.MaxParallelResolutions.ToString());
builder.AppendLine(options.AllowHttpRegistries.ToString());
builder.AppendLine(options.Offline.PreferOffline.ToString());
builder.AppendLine(options.Offline.AllowNetworkFallback.ToString());
foreach (var image in options.Images)
{
builder.AppendLine(image.Reference ?? string.Empty);
builder.AppendLine(image.ExpectedSubjectDigest ?? string.Empty);
builder.AppendLine(image.OfflineBundlePath ?? string.Empty);
}
if (!string.IsNullOrWhiteSpace(options.Offline.RootDirectory))
{
builder.AppendLine(options.Offline.RootDirectory);
builder.AppendLine(options.Offline.DefaultBundleFileName ?? string.Empty);
}
builder.AppendLine(options.Registry.RegistryAuthority ?? string.Empty);
builder.AppendLine(options.Registry.AllowAnonymousFallback.ToString());
var bytes = Encoding.UTF8.GetBytes(builder.ToString());
var hashBytes = sha.ComputeHash(bytes);
return Convert.ToHexString(hashBytes);
}
}

View File

@@ -0,0 +1,6 @@
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciAttestationTarget(
OciImageReference Image,
string? ExpectedSubjectDigest,
OciOfflineBundleReference? OfflineBundle);

View File

@@ -0,0 +1,27 @@
using System;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciImageReference(string Registry, string Repository, string? Tag, string? Digest, string OriginalReference, string Scheme = "https")
{
public string Canonical =>
Digest is not null
? $"{Registry}/{Repository}@{Digest}"
: Tag is not null
? $"{Registry}/{Repository}:{Tag}"
: $"{Registry}/{Repository}";
public bool HasDigest => !string.IsNullOrWhiteSpace(Digest);
public bool HasTag => !string.IsNullOrWhiteSpace(Tag);
public OciImageReference WithDigest(string digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
throw new ArgumentException("Digest must be provided.", nameof(digest));
}
return this with { Digest = digest };
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Text.RegularExpressions;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
internal static class OciImageReferenceParser
{
private static readonly Regex DigestRegex = new(@"^(?<algorithm>[A-Za-z0-9+._-]+):(?<hash>[A-Fa-f0-9]{32,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex RepositoryRegex = new(@"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
public static OciImageReference Parse(string reference)
{
if (string.IsNullOrWhiteSpace(reference))
{
throw new InvalidOperationException("OCI reference cannot be empty.");
}
var trimmed = reference.Trim();
string original = trimmed;
var scheme = "https";
if (trimmed.StartsWith("oci://", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed.Substring("oci://".Length);
}
if (trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed.Substring("https://".Length);
scheme = "https";
}
else if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed.Substring("http://".Length);
scheme = "http";
}
var firstSlash = trimmed.IndexOf('/');
if (firstSlash <= 0)
{
throw new InvalidOperationException($"OCI reference '{reference}' must include a registry and repository component.");
}
var registry = trimmed[..firstSlash];
var remainder = trimmed[(firstSlash + 1)..];
if (!LooksLikeRegistry(registry))
{
throw new InvalidOperationException($"OCI reference '{reference}' is missing an explicit registry component.");
}
string? digest = null;
string? tag = null;
var digestIndex = remainder.IndexOf('@');
if (digestIndex >= 0)
{
digest = remainder[(digestIndex + 1)..];
remainder = remainder[..digestIndex];
if (!DigestRegex.IsMatch(digest))
{
throw new InvalidOperationException($"Digest segment '{digest}' is not a valid OCI digest.");
}
}
var tagIndex = remainder.LastIndexOf(':');
if (tagIndex >= 0)
{
tag = remainder[(tagIndex + 1)..];
remainder = remainder[..tagIndex];
if (string.IsNullOrWhiteSpace(tag))
{
throw new InvalidOperationException("OCI tag segment cannot be empty.");
}
if (tag.Contains('/', StringComparison.Ordinal))
{
throw new InvalidOperationException("OCI tag segment cannot contain '/'.");
}
}
var repository = remainder;
if (string.IsNullOrWhiteSpace(repository))
{
throw new InvalidOperationException("OCI repository segment cannot be empty.");
}
if (!RepositoryRegex.IsMatch(repository))
{
throw new InvalidOperationException($"Repository segment '{repository}' is not valid per OCI distribution rules.");
}
return new OciImageReference(
Registry: registry,
Repository: repository,
Tag: tag,
Digest: digest,
OriginalReference: original,
Scheme: scheme);
}
private static bool LooksLikeRegistry(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (value.Equals("localhost", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (value.Contains('.', StringComparison.Ordinal) || value.Contains(':', StringComparison.Ordinal))
{
return true;
}
// IPv4/IPv6 simplified check
if (value.Length >= 3 && char.IsDigit(value[0]))
{
return true;
}
return false;
}
}

View File

@@ -0,0 +1,5 @@
using System;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
public sealed record OciOfflineBundleReference(string Path, bool Exists, string? ExpectedSubjectDigest);

View File

@@ -0,0 +1,14 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
internal sealed record OciArtifactDescriptor(
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("artifactType")] string? ArtifactType,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations);
internal sealed record OciReferrerIndex(
[property: JsonPropertyName("referrers")] IReadOnlyList<OciArtifactDescriptor> Referrers);

View File

@@ -0,0 +1,13 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
public sealed record OciAttestationDocument(
Uri SourceUri,
ReadOnlyMemory<byte> Content,
ImmutableDictionary<string, string> Metadata,
string? SubjectDigest,
string? ArtifactDigest,
string? ArtifactType,
string SourceKind);

View File

@@ -0,0 +1,259 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Formats.Tar;
using System.IO;
using System.IO.Abstractions;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
public sealed class OciAttestationFetcher
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly IFileSystem _fileSystem;
private readonly ILogger<OciAttestationFetcher> _logger;
public OciAttestationFetcher(
IHttpClientFactory httpClientFactory,
IFileSystem fileSystem,
ILogger<OciAttestationFetcher> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async IAsyncEnumerable<OciAttestationDocument> FetchAsync(
OciAttestationDiscoveryResult discovery,
OciOpenVexAttestationConnectorOptions options,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(discovery);
ArgumentNullException.ThrowIfNull(options);
foreach (var target in discovery.Targets)
{
cancellationToken.ThrowIfCancellationRequested();
bool yieldedOffline = false;
if (target.OfflineBundle is not null && target.OfflineBundle.Exists)
{
await foreach (var offlineDocument in ReadOfflineAsync(target, cancellationToken))
{
yieldedOffline = true;
yield return offlineDocument;
}
if (!discovery.AllowNetworkFallback)
{
continue;
}
}
if (discovery.PreferOffline && yieldedOffline && !discovery.AllowNetworkFallback)
{
continue;
}
if (!discovery.PreferOffline || discovery.AllowNetworkFallback || !yieldedOffline)
{
await foreach (var registryDocument in FetchFromRegistryAsync(discovery, options, target, cancellationToken))
{
yield return registryDocument;
}
}
}
}
private async IAsyncEnumerable<OciAttestationDocument> ReadOfflineAsync(
OciAttestationTarget target,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var offline = target.OfflineBundle!;
var path = _fileSystem.Path.GetFullPath(offline.Path);
if (!_fileSystem.File.Exists(path))
{
if (offline.Exists)
{
_logger.LogWarning("Offline bundle {Path} disappeared before processing.", path);
}
yield break;
}
var extension = _fileSystem.Path.GetExtension(path).ToLowerInvariant();
var subjectDigest = target.Image.Digest ?? target.ExpectedSubjectDigest;
if (string.Equals(extension, ".json", StringComparison.OrdinalIgnoreCase) ||
string.Equals(extension, ".dsse", StringComparison.OrdinalIgnoreCase))
{
var bytes = await _fileSystem.File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
var metadata = BuildOfflineMetadata(target, path, entryName: null, subjectDigest);
yield return new OciAttestationDocument(
new Uri(path, UriKind.Absolute),
bytes,
metadata,
subjectDigest,
null,
null,
"offline");
yield break;
}
if (string.Equals(extension, ".tgz", StringComparison.OrdinalIgnoreCase) ||
string.Equals(extension, ".gz", StringComparison.OrdinalIgnoreCase) ||
string.Equals(extension, ".tar", StringComparison.OrdinalIgnoreCase))
{
await foreach (var document in ReadTarArchiveAsync(target, path, subjectDigest, cancellationToken))
{
yield return document;
}
yield break;
}
// Default: treat as binary blob.
var fallbackBytes = await _fileSystem.File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
var fallbackMetadata = BuildOfflineMetadata(target, path, entryName: null, subjectDigest);
yield return new OciAttestationDocument(
new Uri(path, UriKind.Absolute),
fallbackBytes,
fallbackMetadata,
subjectDigest,
null,
null,
"offline");
}
private async IAsyncEnumerable<OciAttestationDocument> ReadTarArchiveAsync(
OciAttestationTarget target,
string path,
string? subjectDigest,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await using var fileStream = _fileSystem.File.OpenRead(path);
Stream archiveStream = fileStream;
if (path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase) ||
path.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase))
{
archiveStream = new GZipStream(fileStream, CompressionMode.Decompress, leaveOpen: false);
}
using var tarReader = new TarReader(archiveStream, leaveOpen: false);
TarEntry? entry;
while ((entry = await tarReader.GetNextEntryAsync(copyData: false, cancellationToken).ConfigureAwait(false)) is not null)
{
if (entry.EntryType is not TarEntryType.RegularFile || entry.DataStream is null)
{
continue;
}
await using var entryStream = entry.DataStream;
using var buffer = new MemoryStream();
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
var metadata = BuildOfflineMetadata(target, path, entry.Name, subjectDigest);
var sourceUri = new Uri($"{_fileSystem.Path.GetFullPath(path)}#{entry.Name}", UriKind.Absolute);
yield return new OciAttestationDocument(
sourceUri,
buffer.ToArray(),
metadata,
subjectDigest,
null,
null,
"offline");
}
}
private async IAsyncEnumerable<OciAttestationDocument> FetchFromRegistryAsync(
OciAttestationDiscoveryResult discovery,
OciOpenVexAttestationConnectorOptions options,
OciAttestationTarget target,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var registryClient = new OciRegistryClient(
_httpClientFactory,
_logger,
discovery.RegistryAuthorization,
options);
var subjectDigest = target.Image.Digest ?? target.ExpectedSubjectDigest;
if (string.IsNullOrWhiteSpace(subjectDigest))
{
subjectDigest = await registryClient.ResolveDigestAsync(target.Image, cancellationToken).ConfigureAwait(false);
}
if (string.IsNullOrWhiteSpace(subjectDigest))
{
_logger.LogWarning("Unable to resolve subject digest for {Reference}; skipping registry fetch.", target.Image.Canonical);
yield break;
}
if (!string.IsNullOrWhiteSpace(target.ExpectedSubjectDigest) &&
!string.Equals(target.ExpectedSubjectDigest, subjectDigest, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Resolved digest {Resolved} does not match expected digest {Expected} for {Reference}.",
subjectDigest,
target.ExpectedSubjectDigest,
target.Image.Canonical);
}
var descriptors = await registryClient.ListReferrersAsync(target.Image, subjectDigest, cancellationToken).ConfigureAwait(false);
if (descriptors.Count == 0)
{
yield break;
}
foreach (var descriptor in descriptors)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await registryClient.DownloadAttestationAsync(target.Image, descriptor, subjectDigest, cancellationToken).ConfigureAwait(false);
if (document is not null)
{
yield return document;
}
}
}
private static ImmutableDictionary<string, string> BuildOfflineMetadata(
OciAttestationTarget target,
string bundlePath,
string? entryName,
string? subjectDigest)
{
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
builder["oci.image.registry"] = target.Image.Registry;
builder["oci.image.repository"] = target.Image.Repository;
builder["oci.image.reference"] = target.Image.Canonical;
if (!string.IsNullOrWhiteSpace(subjectDigest))
{
builder["oci.image.subjectDigest"] = subjectDigest;
}
if (!string.IsNullOrWhiteSpace(target.ExpectedSubjectDigest))
{
builder["oci.image.expectedSubjectDigest"] = target.ExpectedSubjectDigest!;
}
builder["oci.attestation.sourceKind"] = "offline";
builder["oci.attestation.source"] = bundlePath;
if (!string.IsNullOrWhiteSpace(entryName))
{
builder["oci.attestation.bundleEntry"] = entryName!;
}
return builder.ToImmutable();
}
}

View File

@@ -0,0 +1,363 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
internal sealed class OciRegistryClient
{
private const string ManifestMediaType = "application/vnd.oci.image.manifest.v1+json";
private const string ReferrersArtifactType = "application/vnd.dsse.envelope.v1+json";
private const string DsseMediaType = "application/vnd.dsse.envelope.v1+json";
private const string OpenVexMediaType = "application/vnd.cncf.openvex.v1+json";
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger _logger;
private readonly OciRegistryAuthorization _authorization;
private readonly OciOpenVexAttestationConnectorOptions _options;
public OciRegistryClient(
IHttpClientFactory httpClientFactory,
ILogger logger,
OciRegistryAuthorization authorization,
OciOpenVexAttestationConnectorOptions options)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_authorization = authorization ?? throw new ArgumentNullException(nameof(authorization));
_options = options ?? throw new ArgumentNullException(nameof(options));
}
public async Task<string?> ResolveDigestAsync(OciImageReference image, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(image);
if (image.HasDigest)
{
return image.Digest;
}
var requestUri = BuildRegistryUri(image, $"manifests/{EscapeReference(image.Tag ?? "latest")}");
async Task<HttpRequestMessage> RequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Head, requestUri);
request.Headers.Accept.ParseAdd(ManifestMediaType);
ApplyAuthentication(request);
return await Task.FromResult(request).ConfigureAwait(false);
}
using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("Failed to resolve digest for {Reference}; registry returned 404.", image.Canonical);
return null;
}
response.EnsureSuccessStatusCode();
}
if (response.Headers.TryGetValues("Docker-Content-Digest", out var values))
{
var digest = values.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(digest))
{
return digest;
}
}
// Manifest may have been returned without digest header; fall back to GET.
async Task<HttpRequestMessage> ManifestRequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Accept.ParseAdd(ManifestMediaType);
ApplyAuthentication(request);
return await Task.FromResult(request).ConfigureAwait(false);
}
using var manifestResponse = await SendAsync(ManifestRequestFactory, cancellationToken).ConfigureAwait(false);
manifestResponse.EnsureSuccessStatusCode();
if (manifestResponse.Headers.TryGetValues("Docker-Content-Digest", out var manifestValues))
{
return manifestValues.FirstOrDefault();
}
_logger.LogWarning("Registry {Registry} did not provide Docker-Content-Digest header for {Reference}.", image.Registry, image.Canonical);
return null;
}
public async Task<IReadOnlyList<OciArtifactDescriptor>> ListReferrersAsync(
OciImageReference image,
string subjectDigest,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(image);
ArgumentNullException.ThrowIfNull(subjectDigest);
var query = $"artifactType={Uri.EscapeDataString(ReferrersArtifactType)}";
var requestUri = BuildRegistryUri(image, $"referrers/{subjectDigest}", query);
async Task<HttpRequestMessage> RequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
ApplyAuthentication(request);
request.Headers.Accept.ParseAdd("application/json");
return await Task.FromResult(request).ConfigureAwait(false);
}
using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogDebug("Registry returned 404 for referrers on {Subject}.", subjectDigest);
return Array.Empty<OciArtifactDescriptor>();
}
response.EnsureSuccessStatusCode();
}
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var index = await JsonSerializer.DeserializeAsync<OciReferrerIndex>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
return index?.Referrers ?? Array.Empty<OciArtifactDescriptor>();
}
public async Task<OciAttestationDocument?> DownloadAttestationAsync(
OciImageReference image,
OciArtifactDescriptor descriptor,
string subjectDigest,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(image);
ArgumentNullException.ThrowIfNull(descriptor);
if (!IsSupportedDescriptor(descriptor))
{
return null;
}
var requestUri = BuildRegistryUri(image, $"blobs/{descriptor.Digest}");
async Task<HttpRequestMessage> RequestFactory()
{
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
ApplyAuthentication(request);
request.Headers.Accept.ParseAdd(descriptor.MediaType ?? "application/octet-stream");
return await Task.FromResult(request).ConfigureAwait(false);
}
using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("Registry returned 404 while downloading attestation {Digest} for {Subject}.", descriptor.Digest, subjectDigest);
return null;
}
response.EnsureSuccessStatusCode();
}
var buffer = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(image, descriptor, "registry", requestUri.ToString(), subjectDigest);
return new OciAttestationDocument(
requestUri,
buffer,
metadata,
subjectDigest,
descriptor.Digest,
descriptor.ArtifactType,
"registry");
}
private static bool IsSupportedDescriptor(OciArtifactDescriptor descriptor)
{
if (descriptor is null)
{
return false;
}
if (!string.IsNullOrWhiteSpace(descriptor.ArtifactType) &&
descriptor.ArtifactType.Equals(OpenVexMediaType, StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (!string.IsNullOrWhiteSpace(descriptor.MediaType) &&
(descriptor.MediaType.Equals(DsseMediaType, StringComparison.OrdinalIgnoreCase) ||
descriptor.MediaType.Equals(OpenVexMediaType, StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return false;
}
private async Task<HttpResponseMessage> SendAsync(
Func<Task<HttpRequestMessage>> requestFactory,
CancellationToken cancellationToken)
{
const int maxAttempts = 3;
TimeSpan delay = TimeSpan.FromSeconds(1);
Exception? lastError = null;
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
cancellationToken.ThrowIfCancellationRequested();
using var request = await requestFactory().ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(OciOpenVexAttestationConnectorOptions.HttpClientName);
try
{
var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return response;
}
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
if (_authorization.Mode == OciRegistryAuthMode.Anonymous && !_authorization.AllowAnonymousFallback)
{
var message = $"Registry request to {request.RequestUri} was unauthorized and anonymous fallback is disabled.";
response.Dispose();
throw new HttpRequestException(message);
}
lastError = new HttpRequestException($"Registry returned 401 Unauthorized for {request.RequestUri}.");
}
else if ((int)response.StatusCode >= 500 || response.StatusCode == (HttpStatusCode)429)
{
lastError = new HttpRequestException($"Registry returned status {(int)response.StatusCode} ({response.ReasonPhrase}) for {request.RequestUri}.");
}
else
{
response.EnsureSuccessStatusCode();
}
response.Dispose();
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
lastError = ex;
}
if (attempt < maxAttempts)
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 10));
}
}
throw new HttpRequestException("Failed to execute OCI registry request after multiple attempts.", lastError);
}
private void ApplyAuthentication(HttpRequestMessage request)
{
switch (_authorization.Mode)
{
case OciRegistryAuthMode.Basic when
!string.IsNullOrEmpty(_authorization.Username) &&
!string.IsNullOrEmpty(_authorization.Password):
var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_authorization.Username}:{_authorization.Password}"));
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
break;
case OciRegistryAuthMode.IdentityToken when !string.IsNullOrWhiteSpace(_authorization.IdentityToken):
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authorization.IdentityToken);
break;
case OciRegistryAuthMode.RefreshToken when !string.IsNullOrWhiteSpace(_authorization.RefreshToken):
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authorization.RefreshToken);
break;
default:
if (_authorization.Mode != OciRegistryAuthMode.Anonymous && !_authorization.AllowAnonymousFallback)
{
_logger.LogDebug("No authentication header applied for request to {Uri} (mode {Mode}).", request.RequestUri, _authorization.Mode);
}
break;
}
}
private Uri BuildRegistryUri(OciImageReference image, string relativePath, string? query = null)
{
var scheme = image.Scheme;
if (!string.Equals(scheme, "https", StringComparison.OrdinalIgnoreCase) && !_options.AllowHttpRegistries)
{
throw new InvalidOperationException($"HTTP access to registry '{image.Registry}' is disabled. Set AllowHttpRegistries to true to enable.");
}
var builder = new UriBuilder($"{scheme}://{image.Registry}")
{
Path = $"v2/{BuildRepositoryPath(image.Repository)}/{relativePath}"
};
if (!string.IsNullOrWhiteSpace(query))
{
builder.Query = query;
}
return builder.Uri;
}
private static string BuildRepositoryPath(string repository)
{
var segments = repository.Split('/', StringSplitOptions.RemoveEmptyEntries);
return string.Join('/', segments.Select(Uri.EscapeDataString));
}
private static string EscapeReference(string reference)
{
return Uri.EscapeDataString(reference);
}
private static ImmutableDictionary<string, string> BuildMetadata(
OciImageReference image,
OciArtifactDescriptor descriptor,
string sourceKind,
string sourcePath,
string subjectDigest)
{
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
builder["oci.image.registry"] = image.Registry;
builder["oci.image.repository"] = image.Repository;
builder["oci.image.reference"] = image.Canonical;
builder["oci.image.subjectDigest"] = subjectDigest;
builder["oci.attestation.sourceKind"] = sourceKind;
builder["oci.attestation.source"] = sourcePath;
builder["oci.attestation.artifactDigest"] = descriptor.Digest;
builder["oci.attestation.mediaType"] = descriptor.MediaType ?? string.Empty;
builder["oci.attestation.artifactType"] = descriptor.ArtifactType ?? string.Empty;
builder["oci.attestation.size"] = descriptor.Size.ToString(CultureInfo.InvariantCulture);
if (descriptor.Annotations is not null)
{
foreach (var annotation in descriptor.Annotations)
{
builder[$"oci.attestation.annotations.{annotation.Key}"] = annotation.Value;
}
}
return builder.ToImmutable();
}
}

View File

@@ -0,0 +1,233 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions.Trust;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery;
using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch;
using StellaOps.Excititor.Core;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest;
public sealed class OciOpenVexAttestationConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor StaticDescriptor = new(
id: "excititor:oci.openvex.attest",
kind: VexProviderKind.Attestation,
displayName: "OCI OpenVEX Attestations")
{
Tags = ImmutableArray.Create("oci", "openvex", "attestation", "cosign", "offline"),
};
private readonly OciAttestationDiscoveryService _discoveryService;
private readonly OciAttestationFetcher _fetcher;
private readonly IEnumerable<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>> _validators;
private OciOpenVexAttestationConnectorOptions? _options;
private OciAttestationDiscoveryResult? _discovery;
public OciOpenVexAttestationConnector(
OciAttestationDiscoveryService discoveryService,
OciAttestationFetcher fetcher,
ILogger<OciOpenVexAttestationConnector> logger,
TimeProvider timeProvider,
IEnumerable<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>>? validators = null)
: base(StaticDescriptor, logger, timeProvider)
{
_discoveryService = discoveryService ?? throw new ArgumentNullException(nameof(discoveryService));
_fetcher = fetcher ?? throw new ArgumentNullException(nameof(fetcher));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<OciOpenVexAttestationConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_discovery = await _discoveryService.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Resolved OCI attestation targets.", new Dictionary<string, object?>
{
["targets"] = _discovery.Targets.Length,
["offlinePreferred"] = _discovery.PreferOffline,
["allowNetworkFallback"] = _discovery.AllowNetworkFallback,
["authMode"] = _discovery.RegistryAuthorization.Mode.ToString(),
["cosignMode"] = _discovery.CosignAuthority.Mode.ToString(),
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_discovery is null)
{
_discovery = await _discoveryService.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
}
var documentCount = 0;
await foreach (var document in _fetcher.FetchAsync(_discovery, _options, cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var verificationDocument = CreateRawDocument(
VexDocumentFormat.OciAttestation,
document.SourceUri,
document.Content,
document.Metadata);
var signatureMetadata = await context.SignatureVerifier.VerifyAsync(verificationDocument, cancellationToken).ConfigureAwait(false);
if (signatureMetadata is not null)
{
LogConnectorEvent(LogLevel.Debug, "signature", "Signature metadata captured for attestation.", new Dictionary<string, object?>
{
["subject"] = signatureMetadata.Subject,
["type"] = signatureMetadata.Type,
});
}
var enrichedMetadata = BuildProvenanceMetadata(document, signatureMetadata);
var rawDocument = CreateRawDocument(
VexDocumentFormat.OciAttestation,
document.SourceUri,
document.Content,
enrichedMetadata);
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
documentCount++;
yield return rawDocument;
}
LogConnectorEvent(LogLevel.Information, "fetch", "OCI attestation fetch completed.", new Dictionary<string, object?>
{
["documents"] = documentCount,
["since"] = context.Since?.ToString("O", CultureInfo.InvariantCulture),
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("Attestation documents rely on dedicated normalizers, to be wired in EXCITITOR-CONN-OCI-01-002.");
public OciAttestationDiscoveryResult? GetCachedDiscovery() => _discovery;
private ImmutableDictionary<string, string> BuildProvenanceMetadata(OciAttestationDocument document, VexSignatureMetadata? signature)
{
var builder = document.Metadata.ToBuilder();
if (!string.IsNullOrWhiteSpace(document.SourceKind))
{
builder["vex.provenance.sourceKind"] = document.SourceKind;
}
if (!string.IsNullOrWhiteSpace(document.SubjectDigest))
{
builder["vex.provenance.subjectDigest"] = document.SubjectDigest!;
}
if (!string.IsNullOrWhiteSpace(document.ArtifactDigest))
{
builder["vex.provenance.artifactDigest"] = document.ArtifactDigest!;
}
if (!string.IsNullOrWhiteSpace(document.ArtifactType))
{
builder["vex.provenance.artifactType"] = document.ArtifactType!;
}
if (_discovery is not null)
{
builder["vex.provenance.registryAuthMode"] = _discovery.RegistryAuthorization.Mode.ToString();
var registryAuthority = _discovery.RegistryAuthorization.RegistryAuthority;
if (string.IsNullOrWhiteSpace(registryAuthority))
{
if (builder.TryGetValue("oci.image.registry", out var metadataRegistry) && !string.IsNullOrWhiteSpace(metadataRegistry))
{
registryAuthority = metadataRegistry;
}
}
if (!string.IsNullOrWhiteSpace(registryAuthority))
{
builder["vex.provenance.registryAuthority"] = registryAuthority!;
}
builder["vex.provenance.cosign.mode"] = _discovery.CosignAuthority.Mode.ToString();
if (_discovery.CosignAuthority.Keyless is not null)
{
var keyless = _discovery.CosignAuthority.Keyless;
builder["vex.provenance.cosign.issuer"] = keyless!.Issuer;
builder["vex.provenance.cosign.subject"] = keyless.Subject;
if (keyless.FulcioUrl is not null)
{
builder["vex.provenance.cosign.fulcioUrl"] = keyless.FulcioUrl!.ToString();
}
if (keyless.RekorUrl is not null)
{
builder["vex.provenance.cosign.rekorUrl"] = keyless.RekorUrl!.ToString();
}
}
else if (_discovery.CosignAuthority.KeyPair is not null)
{
var keyPair = _discovery.CosignAuthority.KeyPair;
builder["vex.provenance.cosign.keyPair"] = "true";
if (keyPair!.RekorUrl is not null)
{
builder["vex.provenance.cosign.rekorUrl"] = keyPair.RekorUrl!.ToString();
}
}
}
if (signature is not null)
{
builder["vex.signature.type"] = signature.Type;
if (!string.IsNullOrWhiteSpace(signature.Subject))
{
builder["vex.signature.subject"] = signature.Subject!;
}
if (!string.IsNullOrWhiteSpace(signature.Issuer))
{
builder["vex.signature.issuer"] = signature.Issuer!;
}
if (!string.IsNullOrWhiteSpace(signature.KeyId))
{
builder["vex.signature.keyId"] = signature.KeyId!;
}
if (signature.VerifiedAt is not null)
{
builder["vex.signature.verifiedAt"] = signature.VerifiedAt.Value.ToString("O", CultureInfo.InvariantCulture);
}
if (!string.IsNullOrWhiteSpace(signature.TransparencyLogReference))
{
builder["vex.signature.transparencyLogReference"] = signature.TransparencyLogReference!;
}
}
var metadataBuilder = new VexConnectorMetadataBuilder();
metadataBuilder.AddRange(builder.Select(kv => new KeyValuePair<string, string?>(kv.Key, kv.Value)));
ConnectorSignerMetadataEnricher.Enrich(metadataBuilder, Descriptor.Id, Logger);
foreach (var kv in metadataBuilder.Build())
{
builder[kv.Key] = kv.Value;
}
return builder.ToImmutable();
}
}

View File

@@ -0,0 +1,6 @@
using StellaOps.Plugin.Versioning;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests")]
[assembly: StellaPluginVersion("1.0.0", MinimumHostVersion = "1.0.0", MaximumHostVersion = "1.99.99")]

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="System.IO.Abstractions" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,8 @@
# Completed Tasks
|EXCITITOR-CONN-OCI-01-001 OCI discovery & auth plumbing|Team Excititor Connectors OCI|EXCITITOR-CONN-ABS-01-001|DONE (2025-10-18) Added connector skeleton, options/validators, discovery caching, cosign/auth descriptors, offline bundle resolution, DI wiring, and regression tests.|
|EXCITITOR-CONN-OCI-01-002 Attestation fetch & verify loop|Team Excititor Connectors OCI|EXCITITOR-CONN-OCI-01-001, EXCITITOR-ATTEST-01-002|DONE (2025-10-18) Added offline/registry fetch services, DSSE retrieval with retries, signature verification callout, and raw persistence coverage.|
|EXCITITOR-CONN-OCI-01-003 Provenance metadata & policy hooks|Team Excititor Connectors OCI|EXCITITOR-CONN-OCI-01-002, EXCITITOR-POLICY-01-001|DONE (2025-10-18) Enriched attestation metadata with provenance hints, cosign expectations, registry auth context, and signature diagnostics for policy consumption.|

View File

@@ -0,0 +1,10 @@
# Excititor Connectors OCI OpenVEX Attest Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0302-M | DONE | Revalidated 2026-01-07. |
| AUDIT-0302-T | DONE | Revalidated 2026-01-07. |
| AUDIT-0302-A | TODO | Revalidated 2026-01-07 (open findings; pending approval). |

View File

@@ -0,0 +1,34 @@
# AGENTS
## Role
Connector for Oracle CSAF advisories, including CPU and other bulletin releases, projecting documents into raw storage for normalization.
## Scope
- Discovery of Oracle CSAF catalogue, navigation of quarterly CPU bundles, and delta detection.
- HTTP fetch with retry/backoff, checksum validation, and deduplication across revisions.
- Mapping Oracle advisory metadata (CPU ID, component families) into connector context.
- Publishing trust metadata (PGP keys/cosign options) aligned with policy expectations.
## Participants
- Worker orchestrates regular pulls respecting Oracle publication cadence; WebService offers manual triggers.
- CSAF normalizer processes raw documents to claims.
- Policy engine leverages trust metadata and provenance hints.
## Interfaces & contracts
- Implements `IVexConnector` using shared abstractions for HTTP/resume and telemetry.
- Configuration options for CPU schedule, credentials (if required), and offline snapshot ingestion.
## In/Out of scope
In: fetching, metadata mapping, raw persistence, trust hints.
Out: normalization, storage internals, export/attestation flows.
## Observability & security expectations
- Log CPU release windows, document counts, and fetch durations; redact any secrets.
- Emit metrics for deduped vs new documents and quarantine rates.
## Tests
- Harness tests with mocked Oracle catalogues will live in `../StellaOps.Excititor.Connectors.Oracle.CSAF.Tests`.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,85 @@
using System;
using System.IO;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
public sealed class OracleConnectorOptions
{
public const string HttpClientName = "excititor.connector.oracle.catalog";
/// <summary>
/// Oracle CSAF catalog endpoint hosting advisory metadata.
/// </summary>
public Uri CatalogUri { get; set; } = new("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json");
/// <summary>
/// Optional CPU calendar endpoint providing upcoming release dates.
/// </summary>
public Uri? CpuCalendarUri { get; set; }
/// <summary>
/// Duration the discovery metadata should be cached before refresh.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6);
/// <summary>
/// When true, the loader will prefer offline snapshot data over network fetches.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Optional file path for persisting or ingesting catalog snapshots.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// Enables writing fresh catalog responses to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
/// <summary>
/// Optional request delay when iterating catalogue entries (for rate limiting).
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public void Validate(IFileSystem? fileSystem = null)
{
if (CatalogUri is null || !CatalogUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CatalogUri must be an absolute URI.");
}
if (CatalogUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("CatalogUri must use HTTP or HTTPS.");
}
if (CpuCalendarUri is not null && (!CpuCalendarUri.IsAbsoluteUri || CpuCalendarUri.Scheme is not ("http" or "https")))
{
throw new InvalidOperationException("CpuCalendarUri must be an absolute HTTP(S) URI when provided.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("MetadataCacheDuration must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,33 @@
using StellaOps.Excititor.Connectors.Abstractions;
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
public sealed class OracleConnectorOptionsValidator : IVexConnectorOptionsValidator<OracleConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public OracleConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(VexConnectorDescriptor descriptor, OracleConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,46 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
using StellaOps.Excititor.Core;
using System;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.DependencyInjection;
public static class OracleConnectorServiceCollectionExtensions
{
public static IServiceCollection AddOracleCsafConnector(this IServiceCollection services, Action<OracleConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<OracleConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddSingleton<IVexConnectorOptionsValidator<OracleConnectorOptions>, OracleConnectorOptionsValidator>();
services.AddHttpClient(OracleConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Oracle.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<OracleCatalogLoader>();
services.AddSingleton<IVexConnector, OracleCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,419 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
public sealed class OracleCatalogLoader
{
public const string CachePrefix = "StellaOps.Excititor.Connectors.Oracle.CSAF.Catalog";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<OracleCatalogLoader> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _semaphore = new(1, 1);
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
public OracleCatalogLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IFileSystem fileSystem,
ILogger<OracleCatalogLoader> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<OracleCatalogResult> LoadAsync(OracleConnectorOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
options.Validate(_fileSystem);
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
CacheEntry? entry = null;
if (options.PreferOfflineSnapshot)
{
entry = LoadFromOffline(options);
if (entry is null)
{
throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline snapshot was found or could be loaded.");
}
}
else
{
entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false)
?? LoadFromOffline(options);
}
if (entry is null)
{
throw new InvalidOperationException("Unable to load Oracle CSAF catalog from network or offline snapshot.");
}
var expiration = entry.MetadataCacheDuration == TimeSpan.Zero
? (DateTimeOffset?)null
: _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration);
var cacheEntryOptions = new MemoryCacheEntryOptions();
if (expiration.HasValue)
{
cacheEntryOptions.AbsoluteExpiration = expiration.Value;
}
_memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheEntryOptions);
return entry.ToResult(fromCache: false);
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(OracleConnectorOptions options, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName);
using var response = await client.GetAsync(options.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var catalogPayload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
string? calendarPayload = null;
if (options.CpuCalendarUri is not null)
{
try
{
using var calendarResponse = await client.GetAsync(options.CpuCalendarUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
calendarResponse.EnsureSuccessStatusCode();
calendarPayload = await calendarResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to fetch Oracle CPU calendar from {Uri}; continuing without schedule.", options.CpuCalendarUri);
}
}
var metadata = ParseMetadata(catalogPayload, calendarPayload);
var fetchedAt = _timeProvider.GetUtcNow();
var entry = new CacheEntry(metadata, fetchedAt, fetchedAt, options.MetadataCacheDuration, false);
PersistSnapshotIfNeeded(options, metadata, fetchedAt);
return entry;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to fetch Oracle CSAF catalog from {Uri}; attempting offline fallback if available.", options.CatalogUri);
return null;
}
}
private CacheEntry? LoadFromOffline(OracleConnectorOptions options)
{
if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(options.OfflineSnapshotPath))
{
_logger.LogWarning("Oracle offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath);
var snapshot = JsonSerializer.Deserialize<OracleCatalogSnapshot>(payload, _serializerOptions);
if (snapshot is null)
{
throw new InvalidOperationException("Offline snapshot payload was empty.");
}
return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Oracle CSAF catalog from offline snapshot {Path}.", options.OfflineSnapshotPath);
return null;
}
}
private OracleCatalogMetadata ParseMetadata(string catalogPayload, string? calendarPayload)
{
if (string.IsNullOrWhiteSpace(catalogPayload))
{
throw new InvalidOperationException("Oracle catalog payload was empty.");
}
using var document = JsonDocument.Parse(catalogPayload);
var root = document.RootElement;
var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated)
? generated
: _timeProvider.GetUtcNow();
var entries = ParseEntries(root);
var schedule = ParseSchedule(root);
if (!string.IsNullOrWhiteSpace(calendarPayload))
{
schedule = MergeSchedule(schedule, calendarPayload);
}
return new OracleCatalogMetadata(generatedAt, entries, schedule);
}
private ImmutableArray<OracleCatalogEntry> ParseEntries(JsonElement root)
{
if (!root.TryGetProperty("catalog", out var catalogElement) || catalogElement.ValueKind is not JsonValueKind.Array)
{
return ImmutableArray<OracleCatalogEntry>.Empty;
}
var builder = ImmutableArray.CreateBuilder<OracleCatalogEntry>();
foreach (var entry in catalogElement.EnumerateArray())
{
var id = entry.TryGetProperty("id", out var idElement) && idElement.ValueKind == JsonValueKind.String ? idElement.GetString() : null;
var title = entry.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String ? titleElement.GetString() : null;
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(title))
{
continue;
}
DateTimeOffset publishedAt = default;
if (entry.TryGetProperty("published", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(publishedElement.GetString(), out var publishedParsed))
{
publishedAt = publishedParsed;
}
string? revision = null;
if (entry.TryGetProperty("revision", out var revisionElement) && revisionElement.ValueKind == JsonValueKind.String)
{
revision = revisionElement.GetString();
}
ImmutableArray<string> products = ImmutableArray<string>.Empty;
if (entry.TryGetProperty("products", out var productsElement))
{
products = ParseStringArray(productsElement);
}
Uri? documentUri = null;
string? sha256 = null;
long? size = null;
if (entry.TryGetProperty("document", out var documentElement) && documentElement.ValueKind == JsonValueKind.Object)
{
if (documentElement.TryGetProperty("url", out var urlElement) && urlElement.ValueKind == JsonValueKind.String && Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var parsedUri))
{
documentUri = parsedUri;
}
if (documentElement.TryGetProperty("sha256", out var hashElement) && hashElement.ValueKind == JsonValueKind.String)
{
sha256 = hashElement.GetString();
}
if (documentElement.TryGetProperty("size", out var sizeElement) && sizeElement.ValueKind == JsonValueKind.Number && sizeElement.TryGetInt64(out var parsedSize))
{
size = parsedSize;
}
}
if (documentUri is null)
{
continue;
}
builder.Add(new OracleCatalogEntry(id!, title!, documentUri, publishedAt, revision, sha256, size, products));
}
return builder.ToImmutable();
}
private ImmutableArray<OracleCpuRelease> ParseSchedule(JsonElement root)
{
if (!root.TryGetProperty("schedule", out var scheduleElement) || scheduleElement.ValueKind is not JsonValueKind.Array)
{
return ImmutableArray<OracleCpuRelease>.Empty;
}
var builder = ImmutableArray.CreateBuilder<OracleCpuRelease>();
foreach (var item in scheduleElement.EnumerateArray())
{
var window = item.TryGetProperty("window", out var windowElement) && windowElement.ValueKind == JsonValueKind.String ? windowElement.GetString() : null;
if (string.IsNullOrWhiteSpace(window))
{
continue;
}
DateTimeOffset releaseDate = default;
if (item.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed))
{
releaseDate = parsed;
}
builder.Add(new OracleCpuRelease(window!, releaseDate));
}
return builder.ToImmutable();
}
private ImmutableArray<OracleCpuRelease> MergeSchedule(ImmutableArray<OracleCpuRelease> existing, string calendarPayload)
{
try
{
using var document = JsonDocument.Parse(calendarPayload);
var root = document.RootElement;
if (!root.TryGetProperty("cpuWindows", out var windowsElement) || windowsElement.ValueKind is not JsonValueKind.Array)
{
return existing;
}
var builder = existing.ToBuilder();
var known = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var item in builder)
{
known.Add(item.Window);
}
foreach (var windowElement in windowsElement.EnumerateArray())
{
var name = windowElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
if (!known.Add(name))
{
continue;
}
DateTimeOffset releaseDate = default;
if (windowElement.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed))
{
releaseDate = parsed;
}
builder.Add(new OracleCpuRelease(name!, releaseDate));
}
return builder.ToImmutable();
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to parse Oracle CPU calendar payload; continuing with existing schedule data.");
return existing;
}
}
private ImmutableArray<string> ParseStringArray(JsonElement element)
{
if (element.ValueKind is not JsonValueKind.Array)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var item in element.EnumerateArray())
{
if (item.ValueKind == JsonValueKind.String)
{
var value = item.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
builder.Add(value);
}
}
}
return builder.ToImmutable();
}
private void PersistSnapshotIfNeeded(OracleConnectorOptions options, OracleCatalogMetadata metadata, DateTimeOffset fetchedAt)
{
if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return;
}
try
{
var snapshot = new OracleCatalogSnapshot(metadata, fetchedAt);
var payload = JsonSerializer.Serialize(snapshot, _serializerOptions);
_fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath);
}
}
private static string CreateCacheKey(OracleConnectorOptions options)
=> $"{CachePrefix}:{options.CatalogUri}:{options.CpuCalendarUri}";
private sealed record CacheEntry(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot)
{
public bool IsExpired(DateTimeOffset now)
=> MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration;
public OracleCatalogResult ToResult(bool fromCache)
=> new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot);
}
private sealed record OracleCatalogSnapshot(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt);
}
public sealed record OracleCatalogMetadata(
DateTimeOffset GeneratedAt,
ImmutableArray<OracleCatalogEntry> Entries,
ImmutableArray<OracleCpuRelease> CpuSchedule);
public sealed record OracleCatalogEntry(
string Id,
string Title,
Uri DocumentUri,
DateTimeOffset PublishedAt,
string? Revision,
string? Sha256,
long? Size,
ImmutableArray<string> Products);
public sealed record OracleCpuRelease(string Window, DateTimeOffset ReleaseDate);
public sealed record OracleCatalogResult(
OracleCatalogMetadata Metadata,
DateTimeOffset FetchedAt,
bool FromCache,
bool FromOfflineSnapshot);

View File

@@ -0,0 +1,364 @@
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions.Trust;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF;
public sealed class OracleCsafConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:oracle",
kind: VexProviderKind.Vendor,
displayName: "Oracle CSAF")
{
Tags = ImmutableArray.Create("oracle", "csaf", "cpu"),
};
private readonly OracleCatalogLoader _catalogLoader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> _validators;
private OracleConnectorOptions? _options;
private OracleCatalogResult? _catalog;
public OracleCsafConnector(
OracleCatalogLoader catalogLoader,
IHttpClientFactory httpClientFactory,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> validators,
ILogger<OracleCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<OracleConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary<string, object?>
{
["catalogEntryCount"] = _catalog.Metadata.Entries.Length,
["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length,
["fromOffline"] = _catalog.FromOfflineSnapshot,
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
_catalog ??= await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
var entries = _catalog.Metadata.Entries
.OrderBy(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt)
.ToImmutableArray();
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var since = ResolveSince(context.Since, state?.LastUpdated);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var latestPublished = state?.LastUpdated ?? since ?? DateTimeOffset.MinValue;
var stateChanged = false;
var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName);
LogConnectorEvent(LogLevel.Information, "fetch.begin", "Starting Oracle CSAF catalogue iteration.", new Dictionary<string, object?>
{
["since"] = since?.ToString("O", CultureInfo.InvariantCulture),
["entryCount"] = entries.Length,
});
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
if (ShouldSkipEntry(entry, since))
{
continue;
}
var expectedDigest = NormalizeDigest(entry.Sha256);
if (expectedDigest is not null && digestSet.Contains(expectedDigest))
{
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
LogConnectorEvent(LogLevel.Debug, "fetch.skip.cached", "Skipping Oracle CSAF entry because digest already processed.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["digest"] = expectedDigest,
});
continue;
}
var rawDocument = await DownloadEntryAsync(client, entry, cancellationToken).ConfigureAwait(false);
if (rawDocument is null)
{
continue;
}
if (expectedDigest is not null && !string.Equals(rawDocument.Digest, expectedDigest, StringComparison.OrdinalIgnoreCase))
{
LogConnectorEvent(LogLevel.Warning, "fetch.checksum_mismatch", "Oracle CSAF document checksum mismatch; document skipped.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["expected"] = expectedDigest,
["actual"] = rawDocument.Digest,
["documentUri"] = entry.DocumentUri.ToString(),
});
continue;
}
if (!digestSet.Add(rawDocument.Digest))
{
LogConnectorEvent(LogLevel.Debug, "fetch.skip.duplicate", "Oracle CSAF document digest already ingested.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["digest"] = rawDocument.Digest,
});
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
LogConnectorEvent(LogLevel.Information, "fetch.document_ingested", "Oracle CSAF document stored.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
["digest"] = rawDocument.Digest,
["documentUri"] = entry.DocumentUri.ToString(),
["publishedAt"] = entry.PublishedAt.ToString("O", CultureInfo.InvariantCulture),
});
yield return rawDocument;
if (_options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
if (stateChanged)
{
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latestPublished == DateTimeOffset.MinValue ? baseState.LastUpdated : latestPublished,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
var ingestedCount = digestList.Count - knownDigests.Length;
LogConnectorEvent(LogLevel.Information, "fetch.complete", "Oracle CSAF fetch completed.", new Dictionary<string, object?>
{
["stateChanged"] = stateChanged,
["documentsProcessed"] = ingestedCount,
["latestPublished"] = latestPublished == DateTimeOffset.MinValue ? null : latestPublished.ToString("O", CultureInfo.InvariantCulture),
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers.");
public OracleCatalogResult? GetCachedCatalog() => _catalog;
private static DateTimeOffset? ResolveSince(DateTimeOffset? contextSince, DateTimeOffset? stateSince)
{
if (contextSince is null)
{
return stateSince;
}
if (stateSince is null)
{
return contextSince;
}
return stateSince > contextSince ? stateSince : contextSince;
}
private static bool ShouldSkipEntry(OracleCatalogEntry entry, DateTimeOffset? since)
{
if (since is null)
{
return false;
}
if (entry.PublishedAt == default)
{
return false;
}
return entry.PublishedAt <= since;
}
private async Task<VexRawDocument?> DownloadEntryAsync(HttpClient client, OracleCatalogEntry entry, CancellationToken cancellationToken)
{
if (entry.DocumentUri is null)
{
LogConnectorEvent(LogLevel.Warning, "fetch.skip.missing_uri", "Oracle CSAF entry missing document URI; skipping.", new Dictionary<string, object?>
{
["entryId"] = entry.Id,
});
return null;
}
var payload = await DownloadWithRetryAsync(client, entry.DocumentUri, cancellationToken).ConfigureAwait(false);
if (payload is null)
{
return null;
}
var metadata = BuildMetadata(builder =>
{
builder.Add("oracle.csaf.entryId", entry.Id);
builder.Add("oracle.csaf.title", entry.Title);
builder.Add("oracle.csaf.revision", entry.Revision);
if (entry.PublishedAt != default)
{
builder.Add("oracle.csaf.published", entry.PublishedAt.ToString("O", CultureInfo.InvariantCulture));
}
builder.Add("oracle.csaf.sha256", NormalizeDigest(entry.Sha256));
builder.Add("oracle.csaf.size", entry.Size?.ToString(CultureInfo.InvariantCulture));
if (!entry.Products.IsDefaultOrEmpty)
{
builder.Add("oracle.csaf.products", string.Join(",", entry.Products));
}
ConnectorSignerMetadataEnricher.Enrich(builder, Descriptor.Id, Logger);
});
return CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload.AsMemory(), metadata);
}
private async Task<byte[]?> DownloadWithRetryAsync(HttpClient client, Uri uri, CancellationToken cancellationToken)
{
const int maxAttempts = 3;
var delay = TimeSpan.FromSeconds(1);
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using var response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (IsTransient(response.StatusCode) && attempt < maxAttempts)
{
LogConnectorEvent(LogLevel.Warning, "fetch.retry.status", "Oracle CSAF document request returned transient status; retrying.", new Dictionary<string, object?>
{
["status"] = (int)response.StatusCode,
["attempt"] = attempt,
["uri"] = uri.ToString(),
});
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = delay + delay;
continue;
}
response.EnsureSuccessStatusCode();
}
var bytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
return bytes;
}
catch (Exception ex) when (IsTransient(ex) && attempt < maxAttempts)
{
LogConnectorEvent(LogLevel.Warning, "fetch.retry.exception", "Oracle CSAF document request failed; retrying.", new Dictionary<string, object?>
{
["attempt"] = attempt,
["uri"] = uri.ToString(),
["exception"] = ex.GetType().Name,
});
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = delay + delay;
}
}
LogConnectorEvent(LogLevel.Error, "fetch.failed", "Oracle CSAF document could not be retrieved after retries.", new Dictionary<string, object?>
{
["uri"] = uri.ToString(),
});
return null;
}
private static bool IsTransient(Exception exception)
=> exception is HttpRequestException or IOException or TaskCanceledException;
private static bool IsTransient(HttpStatusCode statusCode)
{
var status = (int)statusCode;
return status is >= 500 or 408 or 429;
}
private static string? NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return null;
}
var trimmed = digest.Trim();
if (!trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
trimmed = "sha256:" + trimmed;
}
return trimmed.ToLowerInvariant();
}
private static DateTimeOffset UpdateLatest(DateTimeOffset current, DateTimeOffset published)
{
if (published == default)
{
return current;
}
return published > current ? published : current;
}
}

View File

@@ -0,0 +1,6 @@
using StellaOps.Plugin.Versioning;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Connectors.Oracle.CSAF.Tests")]
[assembly: StellaPluginVersion("1.0.0", MinimumHostVersion = "1.0.0", MaximumHostVersion = "1.99.99")]

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Persistence\StellaOps.Excititor.Persistence.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="System.IO.Abstractions" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,6 @@
# Completed Tasks
|EXCITITOR-CONN-ORACLE-01-001 Oracle CSAF catalogue discovery|Team Excititor Connectors Oracle|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-19)** Implemented cached Oracle CSAF catalog loader with CPU calendar merge, offline snapshot ingest/persist, options validation + DI wiring, and regression tests; prerequisite EXCITITOR-CONN-ABS-01-001 verified DONE per Sprint 5 log (2025-10-19).|
|EXCITITOR-CONN-ORACLE-01-002 CSAF download & dedupe pipeline|Team Excititor Connectors Oracle|EXCITITOR-CONN-ORACLE-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-19)** Added Oracle CSAF fetch loop with retry/backoff, checksum validation, resume-aware state persistence, digest dedupe, configurable throttling, and raw storage wiring; regression tests cover new ingestion and mismatch handling.|

View File

@@ -0,0 +1,10 @@
# Excititor Connectors Oracle CSAF Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0304-M | DONE | Revalidated 2026-01-07. |
| AUDIT-0304-T | DONE | Revalidated 2026-01-07. |
| AUDIT-0304-A | TODO | Revalidated 2026-01-07 (open findings; pending approval). |

View File

@@ -0,0 +1,36 @@
# AGENTS
## Role
Connector for Red Hat CSAF VEX feeds, fetching provider metadata, CSAF documents, and projecting them into raw storage for normalization.
## Scope
- Discovery via `/.well-known/csaf/provider-metadata.json`, scheduling windows, and ETag-aware HTTP fetches.
- `RedHatProviderMetadataLoader` handles `.well-known` metadata with caching, schema validation, and offline snapshots.
- `RedHatCsafConnector` consumes ROLIE feeds to fetch incremental CSAF documents, honours `context.Since`, and streams raw advisories to storage.
- Mapping Red Hat CSAF specifics (product tree aliases, RHSA identifiers, revision history) into raw documents.
- Emitting structured telemetry and resume markers for incremental pulls.
- Supplying Red Hat-specific trust overrides and provenance hints to normalization.
## Participants
- Worker schedules pulls using this connector; WebService triggers ad-hoc runs.
- CSAF normalizer consumes fetched documents to produce claims.
- Policy/consensus rely on Red Hat trust metadata captured here.
## Interfaces & contracts
- Implements `IVexConnector` with Red Hat-specific options (parallelism, token auth if configured).
- Uses abstractions from `StellaOps.Excititor.Connectors.Abstractions` for HTTP/resume helpers.
## In/Out of scope
In: data acquisition, HTTP retries, raw document persistence, provider metadata population.
Out: normalization, storage internals, attestation, general connector abstractions (covered elsewhere).
## Observability & security expectations
- Log provider metadata URL, revision ids, fetch durations; redact tokens.
- Emit counters for documents fetched, skipped (304), quarantined.
## Tests
- Connector harness tests (mock HTTP) and resume regression cases will live in `../StellaOps.Excititor.Connectors.RedHat.CSAF.Tests`.
## Required Reading
- `docs/modules/excititor/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.

View File

@@ -0,0 +1,104 @@
using System.Collections.Generic;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
public sealed class RedHatConnectorOptions
{
public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json");
/// <summary>
/// HTTP client name registered for the connector.
/// </summary>
public const string HttpClientName = "excititor.connector.redhat";
/// <summary>
/// URI of the CSAF provider metadata document.
/// </summary>
public Uri MetadataUri { get; set; } = DefaultMetadataUri;
/// <summary>
/// Duration to cache loaded metadata before refreshing.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(1);
/// <summary>
/// Optional file path used to store or source offline metadata snapshots.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// When true, the loader prefers the offline snapshot without attempting a network fetch.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Enables writing fresh metadata responses to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
/// <summary>
/// Explicit trust weight override applied to the provider entry.
/// </summary>
public double TrustWeight { get; set; } = 1.0;
/// <summary>
/// Sigstore/Cosign issuer used to verify CSAF signatures, if published.
/// </summary>
public string? CosignIssuer { get; set; } = "https://access.redhat.com";
/// <summary>
/// Identity pattern matched against the Cosign certificate subject.
/// </summary>
public string? CosignIdentityPattern { get; set; } = "^https://access\\.redhat\\.com/.+$";
/// <summary>
/// Optional list of PGP fingerprints recognised for Red Hat CSAF artifacts.
/// </summary>
public IList<string> PgpFingerprints { get; } = new List<string>();
public void Validate(IFileSystem? fileSystem = null)
{
if (MetadataUri is null || !MetadataUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Metadata URI must be absolute.");
}
if (MetadataUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("Metadata URI must use HTTP or HTTPS.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("Metadata cache duration must be positive.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight) || TrustWeight <= 0)
{
TrustWeight = 1.0;
}
else if (TrustWeight > 1.0)
{
TrustWeight = 1.0;
}
if (CosignIssuer is not null)
{
if (string.IsNullOrWhiteSpace(CosignIdentityPattern))
{
throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified.");
}
}
}
}

View File

@@ -0,0 +1,46 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using System.IO.Abstractions;
using System.Net;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
public static class RedHatConnectorServiceCollectionExtensions
{
public static IServiceCollection AddRedHatCsafConnector(this IServiceCollection services, Action<RedHatConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.AddOptions<RedHatConnectorOptions>()
.Configure(options =>
{
configure?.Invoke(options);
})
.PostConfigure(options => options.Validate());
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddHttpClient(RedHatConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.RedHat/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<RedHatProviderMetadataLoader>();
services.AddSingleton<IVexConnector, RedHatCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,313 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Core;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
public sealed class RedHatProviderMetadataLoader
{
public const string CacheKey = "StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
private readonly ILogger<RedHatProviderMetadataLoader> _logger;
private readonly RedHatConnectorOptions _options;
private readonly IFileSystem _fileSystem;
private readonly JsonSerializerOptions _serializerOptions;
private readonly SemaphoreSlim _refreshSemaphore = new(1, 1);
public RedHatProviderMetadataLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IOptions<RedHatConnectorOptions> options,
ILogger<RedHatProviderMetadataLoader> logger,
IFileSystem? fileSystem = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_cache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_fileSystem = fileSystem ?? new FileSystem();
_serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
}
public async Task<RedHatProviderMetadataResult> LoadAsync(CancellationToken cancellationToken)
{
if (_cache.TryGetValue<CacheEntry>(CacheKey, out var cached) && cached is { } cachedEntry && !cachedEntry.IsExpired())
{
_logger.LogDebug("Returning cached Red Hat provider metadata (expires {Expires}).", cachedEntry.ExpiresAt);
return new RedHatProviderMetadataResult(cachedEntry.Provider, cachedEntry.FetchedAt, true, cachedEntry.FromOffline);
}
await _refreshSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_cache.TryGetValue<CacheEntry>(CacheKey, out cached) && cached is { } cachedAfterLock && !cachedAfterLock.IsExpired())
{
return new RedHatProviderMetadataResult(cachedAfterLock.Provider, cachedAfterLock.FetchedAt, true, cachedAfterLock.FromOffline);
}
CacheEntry? previous = cached;
// Attempt live fetch unless offline preferred.
if (!_options.PreferOfflineSnapshot)
{
var httpResult = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false);
if (httpResult is not null)
{
StoreCache(httpResult);
return new RedHatProviderMetadataResult(httpResult.Provider, httpResult.FetchedAt, false, false);
}
}
var offlineResult = TryLoadFromOffline();
if (offlineResult is not null)
{
var offlineEntry = offlineResult with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
FromOffline = true,
};
StoreCache(offlineEntry);
return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true);
}
throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot.");
}
finally
{
_refreshSemaphore.Release();
}
}
private void StoreCache(CacheEntry entry)
{
var cacheEntryOptions = new MemoryCacheEntryOptions
{
AbsoluteExpiration = entry.ExpiresAt,
};
_cache.Set(CacheKey, entry, cacheEntryOptions);
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri);
if (!string.IsNullOrWhiteSpace(previous?.ETag))
{
if (EntityTagHeaderValue.TryParse(previous.ETag, out var etag))
{
request.Headers.IfNoneMatch.Add(etag);
}
}
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotModified && previous is not null)
{
_logger.LogDebug("Red Hat provider metadata not modified (etag {ETag}).", previous.ETag);
return previous with
{
FetchedAt = DateTimeOffset.UtcNow,
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
};
}
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var provider = ParseAndValidate(payload);
var etagHeader = response.Headers.ETag?.ToString();
if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
try
{
_fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
}
}
return new CacheEntry(
provider,
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
etagHeader,
FromOffline: false);
}
catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot)
{
_logger.LogWarning(ex, "Failed to fetch Red Hat provider metadata from {Uri}, will attempt offline snapshot.", _options.MetadataUri);
return null;
}
}
private CacheEntry? TryLoadFromOffline()
{
if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath))
{
_logger.LogWarning("Offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath);
var provider = ParseAndValidate(payload);
return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, ETag: null, FromOffline: true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Red Hat provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath);
return null;
}
}
private VexProvider ParseAndValidate(string payload)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Provider metadata payload was empty.");
}
ProviderMetadataDocument? document;
try
{
document = JsonSerializer.Deserialize<ProviderMetadataDocument>(payload, _serializerOptions);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Provider metadata payload could not be parsed.", ex);
}
if (document is null)
{
throw new InvalidOperationException("Provider metadata payload was null after parsing.");
}
if (document.Metadata?.Provider?.Name is null)
{
throw new InvalidOperationException("Provider metadata missing provider name.");
}
var distributions = document.Distributions?
.Select(static d => d.Directory)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Select(static s => CreateUri(s!, nameof(ProviderMetadataDistribution.Directory)))
.ToImmutableArray() ?? ImmutableArray<Uri>.Empty;
if (distributions.IsDefaultOrEmpty)
{
throw new InvalidOperationException("Provider metadata did not include any valid distribution directories.");
}
Uri? rolieFeed = null;
if (document.Rolie?.Feeds is not null)
{
foreach (var feed in document.Rolie.Feeds)
{
if (!string.IsNullOrWhiteSpace(feed.Url))
{
rolieFeed = CreateUri(feed.Url, "rolie.feeds[].url");
break;
}
}
}
var trust = BuildTrust();
return new VexProvider(
id: "excititor:redhat",
displayName: document.Metadata.Provider.Name,
kind: VexProviderKind.Distro,
baseUris: distributions,
discovery: new VexProviderDiscovery(_options.MetadataUri, rolieFeed),
trust: trust);
}
private VexProviderTrust BuildTrust()
{
VexCosignTrust? cosign = null;
if (!string.IsNullOrWhiteSpace(_options.CosignIssuer) && !string.IsNullOrWhiteSpace(_options.CosignIdentityPattern))
{
cosign = new VexCosignTrust(_options.CosignIssuer!, _options.CosignIdentityPattern!);
}
return new VexProviderTrust(
_options.TrustWeight,
cosign,
_options.PgpFingerprints);
}
private static Uri CreateUri(string value, string propertyName)
{
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException($"Provider metadata field '{propertyName}' must be an absolute HTTP(S) URI.");
}
return uri;
}
private sealed record ProviderMetadataDocument(
[property: JsonPropertyName("metadata")] ProviderMetadata? Metadata,
[property: JsonPropertyName("distributions")] IReadOnlyList<ProviderMetadataDistribution>? Distributions,
[property: JsonPropertyName("rolie")] ProviderMetadataRolie? Rolie);
private sealed record ProviderMetadata(
[property: JsonPropertyName("provider")] ProviderMetadataProvider? Provider);
private sealed record ProviderMetadataProvider(
[property: JsonPropertyName("name")] string? Name);
private sealed record ProviderMetadataDistribution(
[property: JsonPropertyName("directory")] string? Directory);
private sealed record ProviderMetadataRolie(
[property: JsonPropertyName("feeds")] IReadOnlyList<ProviderMetadataRolieFeed>? Feeds);
private sealed record ProviderMetadataRolieFeed(
[property: JsonPropertyName("url")] string? Url);
private sealed record CacheEntry(
VexProvider Provider,
DateTimeOffset FetchedAt,
DateTimeOffset ExpiresAt,
string? ETag,
bool FromOffline)
{
public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt;
}
}
public sealed record RedHatProviderMetadataResult(
VexProvider Provider,
DateTimeOffset FetchedAt,
bool FromCache,
bool FromOfflineSnapshot);

View File

@@ -0,0 +1,6 @@
using StellaOps.Plugin.Versioning;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Connectors.RedHat.CSAF.Tests")]
[assembly: StellaPluginVersion("1.0.0", MinimumHostVersion = "1.0.0", MaximumHostVersion = "1.99.99")]

View File

@@ -0,0 +1,198 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Xml.Linq;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF;
public sealed class RedHatCsafConnector : VexConnectorBase
{
private readonly RedHatProviderMetadataLoader _metadataLoader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IVexConnectorStateRepository _stateRepository;
public RedHatCsafConnector(
VexConnectorDescriptor descriptor,
RedHatProviderMetadataLoader metadataLoader,
IHttpClientFactory httpClientFactory,
IVexConnectorStateRepository stateRepository,
ILogger<RedHatCsafConnector> logger,
TimeProvider timeProvider)
: base(descriptor, logger, timeProvider)
{
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
}
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
// No connector-specific settings yet.
return ValueTask.CompletedTask;
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
if (metadataResult.Provider.Discovery.RolIeService is null)
{
throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed.");
}
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var sinceTimestamp = context.Since;
if (state?.LastUpdated is { } persisted && (sinceTimestamp is null || persisted > sinceTimestamp))
{
sinceTimestamp = persisted;
}
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestList = new List<string>(knownDigests);
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue;
var stateChanged = false;
foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false))
{
if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp)
{
continue;
}
if (entry.DocumentUri is null)
{
Logger.LogDebug("Skipping ROLIE entry {Id} because no document link was provided.", entry.Id);
continue;
}
var rawDocument = await DownloadCsafDocumentAsync(entry, cancellationToken).ConfigureAwait(false);
if (!digestSet.Add(rawDocument.Digest))
{
Logger.LogDebug("Skipping CSAF document {Uri} because digest {Digest} was already processed.", rawDocument.SourceUri, rawDocument.Digest);
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
if (entry.Updated is DateTimeOffset entryUpdated && entryUpdated > latestUpdated)
{
latestUpdated = entryUpdated;
}
yield return rawDocument;
}
if (stateChanged)
{
var newLastUpdated = latestUpdated == DateTimeOffset.MinValue ? state?.LastUpdated : latestUpdated;
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var updatedState = baseState with
{
LastUpdated = newLastUpdated,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false);
}
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
{
// This connector relies on format-specific normalizers registered elsewhere.
throw new NotSupportedException("RedHatCsafConnector does not perform in-line normalization; use the CSAF normalizer component.");
}
private async Task<IReadOnlyList<RolieEntry>> FetchRolieEntriesAsync(Uri feedUri, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var ns = document.Root?.Name.Namespace ?? "http://www.w3.org/2005/Atom";
var entries = document.Root?
.Elements(ns + "entry")
.Select(e => new RolieEntry(
Id: (string?)e.Element(ns + "id"),
Updated: ParseUpdated((string?)e.Element(ns + "updated")),
DocumentUri: ParseDocumentLink(e, ns)))
.Where(entry => entry.Id is not null && entry.Updated is not null)
.OrderBy(entry => entry.Updated)
.ToList() ?? new List<RolieEntry>();
return entries;
}
private static DateTimeOffset? ParseUpdated(string? value)
=> DateTimeOffset.TryParse(value, out var parsed) ? parsed : null;
private static Uri? ParseDocumentLink(XElement entry, XNamespace ns)
{
var linkElements = entry.Elements(ns + "link");
foreach (var link in linkElements)
{
var rel = (string?)link.Attribute("rel");
var href = (string?)link.Attribute("href");
if (string.IsNullOrWhiteSpace(href))
{
continue;
}
if (rel is null || rel.Equals("enclosure", StringComparison.OrdinalIgnoreCase) || rel.Equals("alternate", StringComparison.OrdinalIgnoreCase))
{
if (Uri.TryCreate(href, UriKind.Absolute, out var uri))
{
return uri;
}
}
}
return null;
}
private async Task<VexRawDocument> DownloadCsafDocumentAsync(RolieEntry entry, CancellationToken cancellationToken)
{
var documentUri = entry.DocumentUri ?? throw new InvalidOperationException("ROLIE entry missing document URI.");
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
using var response = await client.GetAsync(documentUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(builder => builder
.Add("redhat.csaf.entryId", entry.Id)
.Add("redhat.csaf.documentUri", documentUri.ToString())
.Add("redhat.csaf.updated", entry.Updated?.ToString("O", CultureInfo.InvariantCulture)));
return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, contentBytes, metadata);
}
private sealed record RolieEntry(string? Id, DateTimeOffset? Updated, Uri? DocumentUri);
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Persistence\StellaOps.Excititor.Persistence.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="System.IO.Abstractions" />
</ItemGroup>
</Project>

Some files were not shown because too many files have changed in this diff Show More