up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,3 +1,3 @@
using Xunit;
[assembly: CollectionBehavior(DisableTestParallelization = true)]
using Xunit;
[assembly: CollectionBehavior(DisableTestParallelization = true)]

View File

@@ -4,48 +4,48 @@ using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.Core.Attestation;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.WebService.Contracts;
public sealed record AdvisoryIngestRequest(
AdvisorySourceRequest Source,
AdvisoryUpstreamRequest Upstream,
AdvisoryContentRequest Content,
AdvisoryIdentifiersRequest Identifiers,
AdvisoryLinksetRequest? Linkset);
public sealed record AdvisorySourceRequest(
[property: JsonPropertyName("vendor")] string Vendor,
[property: JsonPropertyName("connector")] string Connector,
[property: JsonPropertyName("version")] string Version,
[property: JsonPropertyName("stream")] string? Stream);
public sealed record AdvisoryUpstreamRequest(
[property: JsonPropertyName("upstreamId")] string UpstreamId,
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("signature")] AdvisorySignatureRequest Signature,
[property: JsonPropertyName("provenance")] IDictionary<string, string>? Provenance);
public sealed record AdvisorySignatureRequest(
[property: JsonPropertyName("present")] bool Present,
[property: JsonPropertyName("format")] string? Format,
[property: JsonPropertyName("keyId")] string? KeyId,
[property: JsonPropertyName("sig")] string? Signature,
[property: JsonPropertyName("certificate")] string? Certificate,
[property: JsonPropertyName("digest")] string? Digest);
public sealed record AdvisoryContentRequest(
[property: JsonPropertyName("format")] string Format,
[property: JsonPropertyName("specVersion")] string? SpecVersion,
[property: JsonPropertyName("raw")] JsonElement Raw,
[property: JsonPropertyName("encoding")] string? Encoding);
public sealed record AdvisoryIdentifiersRequest(
[property: JsonPropertyName("primary")] string Primary,
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases);
namespace StellaOps.Concelier.WebService.Contracts;
public sealed record AdvisoryIngestRequest(
AdvisorySourceRequest Source,
AdvisoryUpstreamRequest Upstream,
AdvisoryContentRequest Content,
AdvisoryIdentifiersRequest Identifiers,
AdvisoryLinksetRequest? Linkset);
public sealed record AdvisorySourceRequest(
[property: JsonPropertyName("vendor")] string Vendor,
[property: JsonPropertyName("connector")] string Connector,
[property: JsonPropertyName("version")] string Version,
[property: JsonPropertyName("stream")] string? Stream);
public sealed record AdvisoryUpstreamRequest(
[property: JsonPropertyName("upstreamId")] string UpstreamId,
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("signature")] AdvisorySignatureRequest Signature,
[property: JsonPropertyName("provenance")] IDictionary<string, string>? Provenance);
public sealed record AdvisorySignatureRequest(
[property: JsonPropertyName("present")] bool Present,
[property: JsonPropertyName("format")] string? Format,
[property: JsonPropertyName("keyId")] string? KeyId,
[property: JsonPropertyName("sig")] string? Signature,
[property: JsonPropertyName("certificate")] string? Certificate,
[property: JsonPropertyName("digest")] string? Digest);
public sealed record AdvisoryContentRequest(
[property: JsonPropertyName("format")] string Format,
[property: JsonPropertyName("specVersion")] string? SpecVersion,
[property: JsonPropertyName("raw")] JsonElement Raw,
[property: JsonPropertyName("encoding")] string? Encoding);
public sealed record AdvisoryIdentifiersRequest(
[property: JsonPropertyName("primary")] string Primary,
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases);
public sealed record AdvisoryLinksetRequest(
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases,
[property: JsonPropertyName("scopes")] IReadOnlyList<string>? Scopes,
@@ -66,23 +66,23 @@ public sealed record AdvisoryLinksetReferenceRequest(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("source")] string? Source);
public sealed record AdvisoryIngestResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("inserted")] bool Inserted,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("supersedes")] string? Supersedes,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
public sealed record AdvisoryRawRecordResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
public sealed record AdvisoryIngestResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("inserted")] bool Inserted,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("supersedes")] string? Supersedes,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
public sealed record AdvisoryRawRecordResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
public sealed record AdvisoryRawListResponse(
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
@@ -98,44 +98,44 @@ public sealed record AdvisoryRawProvenanceResponse(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
[property: JsonPropertyName("supersedes")] string? Supersedes,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
public sealed record AocVerifyRequest(
[property: JsonPropertyName("since")] DateTimeOffset? Since,
[property: JsonPropertyName("until")] DateTimeOffset? Until,
[property: JsonPropertyName("limit")] int? Limit,
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
public sealed record AocVerifyResponse(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("window")] AocVerifyWindow Window,
[property: JsonPropertyName("checked")] AocVerifyChecked Checked,
[property: JsonPropertyName("violations")] IReadOnlyList<AocVerifyViolation> Violations,
[property: JsonPropertyName("metrics")] AocVerifyMetrics Metrics,
[property: JsonPropertyName("truncated")] bool Truncated);
public sealed record AocVerifyWindow(
[property: JsonPropertyName("from")] DateTimeOffset From,
[property: JsonPropertyName("to")] DateTimeOffset To);
public sealed record AocVerifyChecked(
[property: JsonPropertyName("advisories")] int Advisories,
[property: JsonPropertyName("vex")] int Vex);
public sealed record AocVerifyMetrics(
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
public sealed record AocVerifyViolation(
[property: JsonPropertyName("code")] string Code,
[property: JsonPropertyName("count")] int Count,
[property: JsonPropertyName("examples")] IReadOnlyList<AocVerifyViolationExample> Examples);
public sealed record AocVerifyViolationExample(
[property: JsonPropertyName("source")] string Source,
[property: JsonPropertyName("documentId")] string DocumentId,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("path")] string Path);
[property: JsonPropertyName("supersedes")] string? Supersedes,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
public sealed record AocVerifyRequest(
[property: JsonPropertyName("since")] DateTimeOffset? Since,
[property: JsonPropertyName("until")] DateTimeOffset? Until,
[property: JsonPropertyName("limit")] int? Limit,
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
public sealed record AocVerifyResponse(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("window")] AocVerifyWindow Window,
[property: JsonPropertyName("checked")] AocVerifyChecked Checked,
[property: JsonPropertyName("violations")] IReadOnlyList<AocVerifyViolation> Violations,
[property: JsonPropertyName("metrics")] AocVerifyMetrics Metrics,
[property: JsonPropertyName("truncated")] bool Truncated);
public sealed record AocVerifyWindow(
[property: JsonPropertyName("from")] DateTimeOffset From,
[property: JsonPropertyName("to")] DateTimeOffset To);
public sealed record AocVerifyChecked(
[property: JsonPropertyName("advisories")] int Advisories,
[property: JsonPropertyName("vex")] int Vex);
public sealed record AocVerifyMetrics(
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
public sealed record AocVerifyViolation(
[property: JsonPropertyName("code")] string Code,
[property: JsonPropertyName("count")] int Count,
[property: JsonPropertyName("examples")] IReadOnlyList<AocVerifyViolationExample> Examples);
public sealed record AocVerifyViolationExample(
[property: JsonPropertyName("source")] string Source,
[property: JsonPropertyName("documentId")] string DocumentId,
[property: JsonPropertyName("contentHash")] string ContentHash,
[property: JsonPropertyName("path")] string Path);

View File

@@ -1,27 +1,27 @@
namespace StellaOps.Concelier.WebService.Diagnostics;
internal sealed record StorageHealth(
string Backend,
bool Ready,
DateTimeOffset? CheckedAt,
double? LatencyMs,
string? Error);
internal sealed record TelemetryHealth(
bool Enabled,
bool Tracing,
bool Metrics,
bool Logging);
internal sealed record HealthDocument(
string Status,
DateTimeOffset StartedAt,
double UptimeSeconds,
StorageHealth Storage,
TelemetryHealth Telemetry);
internal sealed record ReadyDocument(
string Status,
DateTimeOffset StartedAt,
double UptimeSeconds,
StorageHealth Storage);
namespace StellaOps.Concelier.WebService.Diagnostics;
internal sealed record StorageHealth(
string Backend,
bool Ready,
DateTimeOffset? CheckedAt,
double? LatencyMs,
string? Error);
internal sealed record TelemetryHealth(
bool Enabled,
bool Tracing,
bool Metrics,
bool Logging);
internal sealed record HealthDocument(
string Status,
DateTimeOffset StartedAt,
double UptimeSeconds,
StorageHealth Storage,
TelemetryHealth Telemetry);
internal sealed record ReadyDocument(
string Status,
DateTimeOffset StartedAt,
double UptimeSeconds,
StorageHealth Storage);

View File

@@ -1,25 +1,25 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.WebService.Diagnostics;
internal static class JobMetrics
{
internal const string MeterName = "StellaOps.Concelier.WebService.Jobs";
private static readonly Meter Meter = new(MeterName);
internal static readonly Counter<long> TriggerCounter = Meter.CreateCounter<long>(
"web.jobs.triggered",
unit: "count",
description: "Number of job trigger requests accepted by the web service.");
internal static readonly Counter<long> TriggerConflictCounter = Meter.CreateCounter<long>(
"web.jobs.trigger.conflict",
unit: "count",
description: "Number of job trigger requests that resulted in conflicts or rejections.");
internal static readonly Counter<long> TriggerFailureCounter = Meter.CreateCounter<long>(
"web.jobs.trigger.failed",
unit: "count",
description: "Number of job trigger requests that failed at runtime.");
}
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.WebService.Diagnostics;
internal static class JobMetrics
{
internal const string MeterName = "StellaOps.Concelier.WebService.Jobs";
private static readonly Meter Meter = new(MeterName);
internal static readonly Counter<long> TriggerCounter = Meter.CreateCounter<long>(
"web.jobs.triggered",
unit: "count",
description: "Number of job trigger requests accepted by the web service.");
internal static readonly Counter<long> TriggerConflictCounter = Meter.CreateCounter<long>(
"web.jobs.trigger.conflict",
unit: "count",
description: "Number of job trigger requests that resulted in conflicts or rejections.");
internal static readonly Counter<long> TriggerFailureCounter = Meter.CreateCounter<long>(
"web.jobs.trigger.failed",
unit: "count",
description: "Number of job trigger requests that failed at runtime.");
}

View File

@@ -1,12 +1,12 @@
namespace StellaOps.Concelier.WebService.Diagnostics;
internal static class ProblemTypes
{
public const string NotFound = "https://stellaops.org/problems/not-found";
public const string Validation = "https://stellaops.org/problems/validation";
public const string Conflict = "https://stellaops.org/problems/conflict";
public const string Locked = "https://stellaops.org/problems/locked";
public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected";
public const string JobFailure = "https://stellaops.org/problems/job-failure";
public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable";
}
namespace StellaOps.Concelier.WebService.Diagnostics;
internal static class ProblemTypes
{
public const string NotFound = "https://stellaops.org/problems/not-found";
public const string Validation = "https://stellaops.org/problems/validation";
public const string Conflict = "https://stellaops.org/problems/conflict";
public const string Locked = "https://stellaops.org/problems/locked";
public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected";
public const string JobFailure = "https://stellaops.org/problems/job-failure";
public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable";
}

View File

@@ -1,74 +1,74 @@
using System.Diagnostics;
namespace StellaOps.Concelier.WebService.Diagnostics;
internal sealed class ServiceStatus
{
private readonly TimeProvider _timeProvider;
private readonly DateTimeOffset _startedAt;
private readonly object _sync = new();
private DateTimeOffset? _bootstrapCompletedAt;
private TimeSpan? _bootstrapDuration;
private DateTimeOffset? _lastReadyCheckAt;
private TimeSpan? _lastStorageLatency;
private string? _lastStorageError;
private bool _lastReadySucceeded;
public ServiceStatus(TimeProvider timeProvider)
{
_timeProvider = timeProvider ?? TimeProvider.System;
_startedAt = _timeProvider.GetUtcNow();
}
public ServiceHealthSnapshot CreateSnapshot()
{
lock (_sync)
{
return new ServiceHealthSnapshot(
CapturedAt: _timeProvider.GetUtcNow(),
StartedAt: _startedAt,
BootstrapCompletedAt: _bootstrapCompletedAt,
BootstrapDuration: _bootstrapDuration,
LastReadyCheckAt: _lastReadyCheckAt,
LastStorageLatency: _lastStorageLatency,
LastStorageError: _lastStorageError,
LastReadySucceeded: _lastReadySucceeded);
}
}
public void MarkBootstrapCompleted(TimeSpan duration)
{
lock (_sync)
{
var completedAt = _timeProvider.GetUtcNow();
_bootstrapCompletedAt = completedAt;
_bootstrapDuration = duration;
_lastReadySucceeded = true;
_lastStorageLatency = duration;
_lastStorageError = null;
_lastReadyCheckAt = completedAt;
}
}
public void RecordStorageCheck(bool success, TimeSpan latency, string? error)
{
lock (_sync)
{
_lastReadySucceeded = success;
_lastStorageLatency = latency;
_lastStorageError = success ? null : error;
_lastReadyCheckAt = _timeProvider.GetUtcNow();
}
}
}
internal sealed record ServiceHealthSnapshot(
DateTimeOffset CapturedAt,
DateTimeOffset StartedAt,
DateTimeOffset? BootstrapCompletedAt,
TimeSpan? BootstrapDuration,
DateTimeOffset? LastReadyCheckAt,
TimeSpan? LastStorageLatency,
string? LastStorageError,
bool LastReadySucceeded);
using System.Diagnostics;
namespace StellaOps.Concelier.WebService.Diagnostics;
internal sealed class ServiceStatus
{
private readonly TimeProvider _timeProvider;
private readonly DateTimeOffset _startedAt;
private readonly object _sync = new();
private DateTimeOffset? _bootstrapCompletedAt;
private TimeSpan? _bootstrapDuration;
private DateTimeOffset? _lastReadyCheckAt;
private TimeSpan? _lastStorageLatency;
private string? _lastStorageError;
private bool _lastReadySucceeded;
public ServiceStatus(TimeProvider timeProvider)
{
_timeProvider = timeProvider ?? TimeProvider.System;
_startedAt = _timeProvider.GetUtcNow();
}
public ServiceHealthSnapshot CreateSnapshot()
{
lock (_sync)
{
return new ServiceHealthSnapshot(
CapturedAt: _timeProvider.GetUtcNow(),
StartedAt: _startedAt,
BootstrapCompletedAt: _bootstrapCompletedAt,
BootstrapDuration: _bootstrapDuration,
LastReadyCheckAt: _lastReadyCheckAt,
LastStorageLatency: _lastStorageLatency,
LastStorageError: _lastStorageError,
LastReadySucceeded: _lastReadySucceeded);
}
}
public void MarkBootstrapCompleted(TimeSpan duration)
{
lock (_sync)
{
var completedAt = _timeProvider.GetUtcNow();
_bootstrapCompletedAt = completedAt;
_bootstrapDuration = duration;
_lastReadySucceeded = true;
_lastStorageLatency = duration;
_lastStorageError = null;
_lastReadyCheckAt = completedAt;
}
}
public void RecordStorageCheck(bool success, TimeSpan latency, string? error)
{
lock (_sync)
{
_lastReadySucceeded = success;
_lastStorageLatency = latency;
_lastStorageError = success ? null : error;
_lastReadyCheckAt = _timeProvider.GetUtcNow();
}
}
}
internal sealed record ServiceHealthSnapshot(
DateTimeOffset CapturedAt,
DateTimeOffset StartedAt,
DateTimeOffset? BootstrapCompletedAt,
TimeSpan? BootstrapDuration,
DateTimeOffset? LastReadyCheckAt,
TimeSpan? LastStorageLatency,
string? LastStorageError,
bool LastReadySucceeded);

View File

@@ -5,16 +5,16 @@ using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.WebService.Contracts;
namespace StellaOps.Concelier.WebService.Extensions;
internal static class AdvisoryRawRequestMapper
{
internal static AdvisoryRawDocument Map(AdvisoryIngestRequest request, string tenant, TimeProvider timeProvider)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentNullException.ThrowIfNull(timeProvider);
var sourceRequest = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
internal static class AdvisoryRawRequestMapper
{
internal static AdvisoryRawDocument Map(AdvisoryIngestRequest request, string tenant, TimeProvider timeProvider)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentNullException.ThrowIfNull(timeProvider);
var sourceRequest = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
var upstreamRequest = request.Upstream ?? throw new ArgumentException("upstream section is required.", nameof(request));
var contentRequest = request.Content ?? throw new ArgumentException("content section is required.", nameof(request));
var identifiersRequest = request.Identifiers ?? throw new ArgumentException("identifiers section is required.", nameof(request));
@@ -22,18 +22,18 @@ internal static class AdvisoryRawRequestMapper
var source = new RawSourceMetadata(
sourceRequest.Vendor,
sourceRequest.Connector,
sourceRequest.Version,
string.IsNullOrWhiteSpace(sourceRequest.Stream) ? null : sourceRequest.Stream);
var signatureRequest = upstreamRequest.Signature ?? new AdvisorySignatureRequest(false, null, null, null, null, null);
var signature = new RawSignatureMetadata(
signatureRequest.Present,
string.IsNullOrWhiteSpace(signatureRequest.Format) ? null : signatureRequest.Format,
string.IsNullOrWhiteSpace(signatureRequest.KeyId) ? null : signatureRequest.KeyId,
string.IsNullOrWhiteSpace(signatureRequest.Signature) ? null : signatureRequest.Signature,
string.IsNullOrWhiteSpace(signatureRequest.Certificate) ? null : signatureRequest.Certificate,
string.IsNullOrWhiteSpace(signatureRequest.Digest) ? null : signatureRequest.Digest);
sourceRequest.Version,
string.IsNullOrWhiteSpace(sourceRequest.Stream) ? null : sourceRequest.Stream);
var signatureRequest = upstreamRequest.Signature ?? new AdvisorySignatureRequest(false, null, null, null, null, null);
var signature = new RawSignatureMetadata(
signatureRequest.Present,
string.IsNullOrWhiteSpace(signatureRequest.Format) ? null : signatureRequest.Format,
string.IsNullOrWhiteSpace(signatureRequest.KeyId) ? null : signatureRequest.KeyId,
string.IsNullOrWhiteSpace(signatureRequest.Signature) ? null : signatureRequest.Signature,
string.IsNullOrWhiteSpace(signatureRequest.Certificate) ? null : signatureRequest.Certificate,
string.IsNullOrWhiteSpace(signatureRequest.Digest) ? null : signatureRequest.Digest);
var retrievedAt = upstreamRequest.RetrievedAt ?? timeProvider.GetUtcNow();
var upstream = new RawUpstreamMetadata(
upstreamRequest.UpstreamId,
@@ -49,13 +49,13 @@ internal static class AdvisoryRawRequestMapper
string.IsNullOrWhiteSpace(contentRequest.SpecVersion) ? null : contentRequest.SpecVersion,
rawContent,
string.IsNullOrWhiteSpace(contentRequest.Encoding) ? null : contentRequest.Encoding);
var aliases = NormalizeStrings(identifiersRequest.Aliases);
if (aliases.IsDefault)
{
aliases = ImmutableArray<string>.Empty;
}
var aliases = NormalizeStrings(identifiersRequest.Aliases);
if (aliases.IsDefault)
{
aliases = ImmutableArray<string>.Empty;
}
var identifiers = new RawIdentifiers(
aliases,
identifiersRequest.Primary);
@@ -89,70 +89,70 @@ internal static class AdvisoryRawRequestMapper
AdvisoryKey: advisoryKey,
Links: links);
}
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
{
if (values is null)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
internal static ImmutableDictionary<string, string> NormalizeDictionary(IDictionary<string, string>? values)
{
if (values is null || values.Count == 0)
{
return ImmutableDictionary<string, string>.Empty;
}
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var kv in values)
{
if (string.IsNullOrWhiteSpace(kv.Key))
{
continue;
}
builder[kv.Key.Trim()] = kv.Value?.Trim() ?? string.Empty;
}
return builder.ToImmutable();
}
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
{
if (values is null)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
internal static ImmutableDictionary<string, string> NormalizeDictionary(IDictionary<string, string>? values)
{
if (values is null || values.Count == 0)
{
return ImmutableDictionary<string, string>.Empty;
}
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var kv in values)
{
if (string.IsNullOrWhiteSpace(kv.Key))
{
continue;
}
builder[kv.Key.Trim()] = kv.Value?.Trim() ?? string.Empty;
}
return builder.ToImmutable();
}
private static ImmutableArray<RawReference> NormalizeReferences(IEnumerable<AdvisoryLinksetReferenceRequest>? references)
{
if (references is null)
{
return ImmutableArray<RawReference>.Empty;
}
var builder = ImmutableArray.CreateBuilder<RawReference>();
foreach (var reference in references)
{
if (reference is null)
{
continue;
}
if (string.IsNullOrWhiteSpace(reference.Type) || string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
builder.Add(new RawReference(reference.Type.Trim(), reference.Url.Trim(), string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
var builder = ImmutableArray.CreateBuilder<RawReference>();
foreach (var reference in references)
{
if (reference is null)
{
continue;
}
if (string.IsNullOrWhiteSpace(reference.Type) || string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
builder.Add(new RawReference(reference.Type.Trim(), reference.Url.Trim(), string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
}
return builder.Count == 0 ? ImmutableArray<RawReference>.Empty : builder.ToImmutable();
@@ -185,7 +185,7 @@ internal static class AdvisoryRawRequestMapper
return builder.Count == 0 ? ImmutableArray<RawRelationship>.Empty : builder.ToImmutable();
}
private static JsonElement NormalizeRawContent(JsonElement element)
{
var json = element.ValueKind == JsonValueKind.Undefined ? "{}" : element.GetRawText();

View File

@@ -1,38 +1,38 @@
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Configuration;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace StellaOps.Concelier.WebService.Extensions;
public static class ConfigurationExtensions
{
public static IConfigurationBuilder AddConcelierYaml(this IConfigurationBuilder builder, string path)
{
if (builder is null)
{
throw new ArgumentNullException(nameof(builder));
}
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
{
return builder;
}
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
using var reader = File.OpenText(path);
var yamlObject = deserializer.Deserialize(reader);
if (yamlObject is null)
{
return builder;
}
var json = JsonSerializer.Serialize(yamlObject);
var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
return builder.AddJsonStream(stream);
}
}
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Configuration;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace StellaOps.Concelier.WebService.Extensions;
public static class ConfigurationExtensions
{
public static IConfigurationBuilder AddConcelierYaml(this IConfigurationBuilder builder, string path)
{
if (builder is null)
{
throw new ArgumentNullException(nameof(builder));
}
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
{
return builder;
}
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
using var reader = File.OpenText(path);
var yamlObject = deserializer.Deserialize(reader);
if (yamlObject is null)
{
return builder;
}
var json = JsonSerializer.Serialize(yamlObject);
var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
return builder.AddJsonStream(stream);
}
}

View File

@@ -1,208 +1,208 @@
using System.Globalization;
using System.IO;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.WebService.Diagnostics;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.WebService.Services;
using StellaOps.Concelier.WebService.Results;
using HttpResults = Microsoft.AspNetCore.Http.Results;
namespace StellaOps.Concelier.WebService.Extensions;
internal static class MirrorEndpointExtensions
{
private const string IndexScope = "index";
private const string DownloadScope = "download";
public static void MapConcelierMirrorEndpoints(this WebApplication app, bool authorityConfigured, bool enforceAuthority)
{
app.MapGet("/concelier/exports/index.json", async (
MirrorFileLocator locator,
MirrorRateLimiter limiter,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
HttpContext context,
CancellationToken cancellationToken) =>
{
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirrorOptions.Enabled)
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
{
return unauthorizedResult;
}
if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter))
{
ApplyRetryAfter(context.Response, retryAfter);
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
}
if (!locator.TryResolveIndex(out var path, out _))
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
return await WriteFileAsync(context, path, "application/json").ConfigureAwait(false);
});
app.MapGet("/concelier/exports/{**relativePath}", async (
string? relativePath,
MirrorFileLocator locator,
MirrorRateLimiter limiter,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
HttpContext context,
CancellationToken cancellationToken) =>
{
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirrorOptions.Enabled)
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (string.IsNullOrWhiteSpace(relativePath))
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId))
{
return ConcelierProblemResultFactory.MirrorNotFound(context, relativePath);
}
var domain = FindDomain(mirrorOptions, domainId);
if (!TryAuthorize(domain?.RequireAuthentication ?? mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
{
return unauthorizedResult;
}
var limit = domain?.MaxDownloadRequestsPerHour ?? mirrorOptions.MaxIndexRequestsPerHour;
if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter))
{
ApplyRetryAfter(context.Response, retryAfter);
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
}
var contentType = ResolveContentType(path);
return await WriteFileAsync(context, path, contentType).ConfigureAwait(false);
});
}
private static ConcelierOptions.MirrorDomainOptions? FindDomain(ConcelierOptions.MirrorOptions mirrorOptions, string? domainId)
{
if (domainId is null)
{
return null;
}
foreach (var candidate in mirrorOptions.Domains)
{
if (candidate is null)
{
continue;
}
if (string.Equals(candidate.Id, domainId, StringComparison.OrdinalIgnoreCase))
{
return candidate;
}
}
return null;
}
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
{
result = HttpResults.Empty;
if (!requireAuthentication)
{
return true;
}
if (!enforceAuthority || !authorityConfigured)
{
return true;
}
if (context.User?.Identity?.IsAuthenticated == true)
{
return true;
}
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
result = HttpResults.StatusCode(StatusCodes.Status401Unauthorized);
return false;
}
private static Task<IResult> WriteFileAsync(HttpContext context, string path, string contentType)
{
var fileInfo = new FileInfo(path);
if (!fileInfo.Exists)
{
return Task.FromResult(ConcelierProblemResultFactory.MirrorNotFound(context, path));
}
var stream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete);
context.Response.Headers.CacheControl = BuildCacheControlHeader(path);
context.Response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
context.Response.ContentLength = fileInfo.Length;
return Task.FromResult(HttpResults.Stream(stream, contentType));
}
private static string ResolveContentType(string path)
{
if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
{
return "application/json";
}
if (path.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
{
return "application/jose+json";
}
return "application/octet-stream";
}
private static void ApplyRetryAfter(HttpResponse response, TimeSpan? retryAfter)
{
if (retryAfter is null)
{
return;
}
var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1);
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
}
private static string BuildCacheControlHeader(string path)
{
var fileName = Path.GetFileName(path);
if (fileName is null)
{
return "public, max-age=60";
}
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=60";
}
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=300, immutable";
}
return "public, max-age=300";
}
}
using System.Globalization;
using System.IO;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.WebService.Diagnostics;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.WebService.Services;
using StellaOps.Concelier.WebService.Results;
using HttpResults = Microsoft.AspNetCore.Http.Results;
namespace StellaOps.Concelier.WebService.Extensions;
internal static class MirrorEndpointExtensions
{
private const string IndexScope = "index";
private const string DownloadScope = "download";
public static void MapConcelierMirrorEndpoints(this WebApplication app, bool authorityConfigured, bool enforceAuthority)
{
app.MapGet("/concelier/exports/index.json", async (
MirrorFileLocator locator,
MirrorRateLimiter limiter,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
HttpContext context,
CancellationToken cancellationToken) =>
{
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirrorOptions.Enabled)
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
{
return unauthorizedResult;
}
if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter))
{
ApplyRetryAfter(context.Response, retryAfter);
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
}
if (!locator.TryResolveIndex(out var path, out _))
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
return await WriteFileAsync(context, path, "application/json").ConfigureAwait(false);
});
app.MapGet("/concelier/exports/{**relativePath}", async (
string? relativePath,
MirrorFileLocator locator,
MirrorRateLimiter limiter,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
HttpContext context,
CancellationToken cancellationToken) =>
{
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirrorOptions.Enabled)
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (string.IsNullOrWhiteSpace(relativePath))
{
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId))
{
return ConcelierProblemResultFactory.MirrorNotFound(context, relativePath);
}
var domain = FindDomain(mirrorOptions, domainId);
if (!TryAuthorize(domain?.RequireAuthentication ?? mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
{
return unauthorizedResult;
}
var limit = domain?.MaxDownloadRequestsPerHour ?? mirrorOptions.MaxIndexRequestsPerHour;
if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter))
{
ApplyRetryAfter(context.Response, retryAfter);
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
}
var contentType = ResolveContentType(path);
return await WriteFileAsync(context, path, contentType).ConfigureAwait(false);
});
}
private static ConcelierOptions.MirrorDomainOptions? FindDomain(ConcelierOptions.MirrorOptions mirrorOptions, string? domainId)
{
if (domainId is null)
{
return null;
}
foreach (var candidate in mirrorOptions.Domains)
{
if (candidate is null)
{
continue;
}
if (string.Equals(candidate.Id, domainId, StringComparison.OrdinalIgnoreCase))
{
return candidate;
}
}
return null;
}
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
{
result = HttpResults.Empty;
if (!requireAuthentication)
{
return true;
}
if (!enforceAuthority || !authorityConfigured)
{
return true;
}
if (context.User?.Identity?.IsAuthenticated == true)
{
return true;
}
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
result = HttpResults.StatusCode(StatusCodes.Status401Unauthorized);
return false;
}
private static Task<IResult> WriteFileAsync(HttpContext context, string path, string contentType)
{
var fileInfo = new FileInfo(path);
if (!fileInfo.Exists)
{
return Task.FromResult(ConcelierProblemResultFactory.MirrorNotFound(context, path));
}
var stream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete);
context.Response.Headers.CacheControl = BuildCacheControlHeader(path);
context.Response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
context.Response.ContentLength = fileInfo.Length;
return Task.FromResult(HttpResults.Stream(stream, contentType));
}
private static string ResolveContentType(string path)
{
if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
{
return "application/json";
}
if (path.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
{
return "application/jose+json";
}
return "application/octet-stream";
}
private static void ApplyRetryAfter(HttpResponse response, TimeSpan? retryAfter)
{
if (retryAfter is null)
{
return;
}
var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1);
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
}
private static string BuildCacheControlHeader(string path)
{
var fileName = Path.GetFileName(path);
if (fileName is null)
{
return "public, max-age=60";
}
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=60";
}
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=300, immutable";
}
return "public, max-age=300";
}
}

View File

@@ -1,104 +1,104 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Security.Claims;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Auth.Abstractions;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Filters;
/// <summary>
/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage.
/// </summary>
public sealed class JobAuthorizationAuditFilter : IEndpointFilter
{
internal const string LoggerName = "Concelier.Authorization.Audit";
public async ValueTask<object?> InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(next);
var httpContext = context.HttpContext;
var options = httpContext.RequestServices.GetRequiredService<IOptions<ConcelierOptions>>().Value;
var authority = options.Authority;
if (authority is null || !authority.Enabled)
{
return await next(context).ConfigureAwait(false);
}
var logger = httpContext.RequestServices
.GetRequiredService<ILoggerFactory>()
.CreateLogger(LoggerName);
var remoteAddress = httpContext.Connection.RemoteIpAddress;
var matcher = new NetworkMaskMatcher(authority.BypassNetworks);
var user = httpContext.User;
var isAuthenticated = user?.Identity?.IsAuthenticated ?? false;
var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress);
var result = await next(context).ConfigureAwait(false);
var scopes = ExtractScopes(user);
var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value;
var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value;
logger.LogInformation(
"Concelier authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}",
httpContext.Request.Path.Value ?? string.Empty,
httpContext.Response.StatusCode,
string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject,
string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId,
scopes.Length == 0 ? "(none)" : string.Join(',', scopes),
bypassUsed,
remoteAddress?.ToString() ?? IPAddress.None.ToString());
return result;
}
private static string[] ExtractScopes(ClaimsPrincipal? principal)
{
if (principal is null)
{
return Array.Empty<string>();
}
var values = new HashSet<string>(StringComparer.Ordinal);
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem))
{
if (string.IsNullOrWhiteSpace(claim.Value))
{
continue;
}
values.Add(claim.Value);
}
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope))
{
if (string.IsNullOrWhiteSpace(claim.Value))
{
continue;
}
var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
foreach (var part in parts)
{
var normalized = StellaOpsScopes.Normalize(part);
if (!string.IsNullOrEmpty(normalized))
{
values.Add(normalized);
}
}
}
return values.Count == 0 ? Array.Empty<string>() : values.ToArray();
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Security.Claims;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Auth.Abstractions;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Filters;
/// <summary>
/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage.
/// </summary>
public sealed class JobAuthorizationAuditFilter : IEndpointFilter
{
internal const string LoggerName = "Concelier.Authorization.Audit";
public async ValueTask<object?> InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(next);
var httpContext = context.HttpContext;
var options = httpContext.RequestServices.GetRequiredService<IOptions<ConcelierOptions>>().Value;
var authority = options.Authority;
if (authority is null || !authority.Enabled)
{
return await next(context).ConfigureAwait(false);
}
var logger = httpContext.RequestServices
.GetRequiredService<ILoggerFactory>()
.CreateLogger(LoggerName);
var remoteAddress = httpContext.Connection.RemoteIpAddress;
var matcher = new NetworkMaskMatcher(authority.BypassNetworks);
var user = httpContext.User;
var isAuthenticated = user?.Identity?.IsAuthenticated ?? false;
var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress);
var result = await next(context).ConfigureAwait(false);
var scopes = ExtractScopes(user);
var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value;
var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value;
logger.LogInformation(
"Concelier authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}",
httpContext.Request.Path.Value ?? string.Empty,
httpContext.Response.StatusCode,
string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject,
string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId,
scopes.Length == 0 ? "(none)" : string.Join(',', scopes),
bypassUsed,
remoteAddress?.ToString() ?? IPAddress.None.ToString());
return result;
}
private static string[] ExtractScopes(ClaimsPrincipal? principal)
{
if (principal is null)
{
return Array.Empty<string>();
}
var values = new HashSet<string>(StringComparer.Ordinal);
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem))
{
if (string.IsNullOrWhiteSpace(claim.Value))
{
continue;
}
values.Add(claim.Value);
}
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope))
{
if (string.IsNullOrWhiteSpace(claim.Value))
{
continue;
}
var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
foreach (var part in parts)
{
var normalized = StellaOpsScopes.Normalize(part);
if (!string.IsNullOrEmpty(normalized))
{
values.Add(normalized);
}
}
}
return values.Count == 0 ? Array.Empty<string>() : values.ToArray();
}
}

View File

@@ -1,23 +1,23 @@
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.WebService.Jobs;
public sealed record JobDefinitionResponse(
string Kind,
bool Enabled,
string? CronExpression,
TimeSpan Timeout,
TimeSpan LeaseDuration,
JobRunResponse? LastRun)
{
public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun)
{
return new JobDefinitionResponse(
definition.Kind,
definition.Enabled,
definition.CronExpression,
definition.Timeout,
definition.LeaseDuration,
lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun));
}
}
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.WebService.Jobs;
public sealed record JobDefinitionResponse(
string Kind,
bool Enabled,
string? CronExpression,
TimeSpan Timeout,
TimeSpan LeaseDuration,
JobRunResponse? LastRun)
{
public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun)
{
return new JobDefinitionResponse(
definition.Kind,
definition.Enabled,
definition.CronExpression,
definition.Timeout,
definition.LeaseDuration,
lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun));
}
}

View File

@@ -1,29 +1,29 @@
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.WebService.Jobs;
public sealed record JobRunResponse(
Guid RunId,
string Kind,
JobRunStatus Status,
string Trigger,
DateTimeOffset CreatedAt,
DateTimeOffset? StartedAt,
DateTimeOffset? CompletedAt,
string? Error,
TimeSpan? Duration,
IReadOnlyDictionary<string, object?> Parameters)
{
public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot)
=> new(
snapshot.RunId,
snapshot.Kind,
snapshot.Status,
snapshot.Trigger,
snapshot.CreatedAt,
snapshot.StartedAt,
snapshot.CompletedAt,
snapshot.Error,
snapshot.Duration,
snapshot.Parameters);
}
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.WebService.Jobs;
public sealed record JobRunResponse(
Guid RunId,
string Kind,
JobRunStatus Status,
string Trigger,
DateTimeOffset CreatedAt,
DateTimeOffset? StartedAt,
DateTimeOffset? CompletedAt,
string? Error,
TimeSpan? Duration,
IReadOnlyDictionary<string, object?> Parameters)
{
public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot)
=> new(
snapshot.RunId,
snapshot.Kind,
snapshot.Status,
snapshot.Trigger,
snapshot.CreatedAt,
snapshot.StartedAt,
snapshot.CompletedAt,
snapshot.Error,
snapshot.Duration,
snapshot.Parameters);
}

View File

@@ -1,8 +1,8 @@
namespace StellaOps.Concelier.WebService.Jobs;
public sealed class JobTriggerRequest
{
public string Trigger { get; set; } = "api";
public Dictionary<string, object?> Parameters { get; set; } = new(StringComparer.Ordinal);
}
namespace StellaOps.Concelier.WebService.Jobs;
public sealed class JobTriggerRequest
{
public string Trigger { get; set; } = "api";
public Dictionary<string, object?> Parameters { get; set; } = new(StringComparer.Ordinal);
}

View File

@@ -1,71 +1,71 @@
using System;
using System.IO;
namespace StellaOps.Concelier.WebService.Options;
/// <summary>
/// Post-configuration helpers for <see cref="ConcelierOptions"/>.
/// </summary>
public static class ConcelierOptionsPostConfigure
{
/// <summary>
/// Applies derived settings that require filesystem access, such as loading client secrets from disk.
/// </summary>
/// <param name="options">The options to mutate.</param>
/// <param name="contentRootPath">Application content root used to resolve relative paths.</param>
public static void Apply(ConcelierOptions options, string contentRootPath)
{
ArgumentNullException.ThrowIfNull(options);
using System;
using System.IO;
namespace StellaOps.Concelier.WebService.Options;
/// <summary>
/// Post-configuration helpers for <see cref="ConcelierOptions"/>.
/// </summary>
public static class ConcelierOptionsPostConfigure
{
/// <summary>
/// Applies derived settings that require filesystem access, such as loading client secrets from disk.
/// </summary>
/// <param name="options">The options to mutate.</param>
/// <param name="contentRootPath">Application content root used to resolve relative paths.</param>
public static void Apply(ConcelierOptions options, string contentRootPath)
{
ArgumentNullException.ThrowIfNull(options);
options.Authority ??= new ConcelierOptions.AuthorityOptions();
options.Features ??= new ConcelierOptions.FeaturesOptions();
options.Evidence ??= new ConcelierOptions.EvidenceBundleOptions();
var authority = options.Authority;
if (string.IsNullOrWhiteSpace(authority.ClientSecret)
&& !string.IsNullOrWhiteSpace(authority.ClientSecretFile))
{
var resolvedPath = authority.ClientSecretFile!;
if (!Path.IsPathRooted(resolvedPath))
{
resolvedPath = Path.Combine(contentRootPath, resolvedPath);
}
if (!File.Exists(resolvedPath))
{
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found.");
}
var secret = File.ReadAllText(resolvedPath).Trim();
if (string.IsNullOrEmpty(secret))
{
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty.");
}
authority.ClientSecret = secret;
}
var authority = options.Authority;
if (string.IsNullOrWhiteSpace(authority.ClientSecret)
&& !string.IsNullOrWhiteSpace(authority.ClientSecretFile))
{
var resolvedPath = authority.ClientSecretFile!;
if (!Path.IsPathRooted(resolvedPath))
{
resolvedPath = Path.Combine(contentRootPath, resolvedPath);
}
if (!File.Exists(resolvedPath))
{
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found.");
}
var secret = File.ReadAllText(resolvedPath).Trim();
if (string.IsNullOrEmpty(secret))
{
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty.");
}
authority.ClientSecret = secret;
}
options.Mirror ??= new ConcelierOptions.MirrorOptions();
var mirror = options.Mirror;
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
{
mirror.ExportRoot = Path.Combine("exports", "json");
}
var resolvedRoot = mirror.ExportRoot;
if (!Path.IsPathRooted(resolvedRoot))
{
resolvedRoot = Path.Combine(contentRootPath, resolvedRoot);
}
mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot);
if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName))
{
mirror.LatestDirectoryName = "latest";
}
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
{
mirror.ExportRoot = Path.Combine("exports", "json");
}
var resolvedRoot = mirror.ExportRoot;
if (!Path.IsPathRooted(resolvedRoot))
{
resolvedRoot = Path.Combine(contentRootPath, resolvedRoot);
}
mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot);
if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName))
{
mirror.LatestDirectoryName = "latest";
}
if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName))
{
mirror.MirrorDirectoryName = "mirror";

View File

@@ -65,7 +65,7 @@ using StellaOps.Aoc.AspNetCore.Results;
using HttpResults = Microsoft.AspNetCore.Http.Results;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage.Aliases;
using StellaOps.Provenance.Mongo;
using StellaOps.Provenance;
namespace StellaOps.Concelier.WebService
{

View File

@@ -1,184 +1,184 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Services;
internal sealed class MirrorFileLocator
{
private readonly IOptionsMonitor<ConcelierOptions> _options;
private readonly ILogger<MirrorFileLocator> _logger;
public MirrorFileLocator(IOptionsMonitor<ConcelierOptions> options, ILogger<MirrorFileLocator> logger)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public bool TryResolveIndex([NotNullWhen(true)] out string? path, [NotNullWhen(true)] out string? exportId)
=> TryResolveRelativePath("index.json", out path, out exportId, out _);
public bool TryResolveRelativePath(string relativePath, [NotNullWhen(true)] out string? fullPath, [NotNullWhen(true)] out string? exportId, out string? domainId)
{
fullPath = null;
exportId = null;
domainId = null;
var mirror = _options.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirror.Enabled)
{
return false;
}
if (!TryResolveExportDirectory(mirror, out var exportDirectory, out exportId))
{
return false;
}
var sanitized = SanitizeRelativePath(relativePath);
if (sanitized.Length == 0 || string.Equals(sanitized, "index.json", StringComparison.OrdinalIgnoreCase))
{
sanitized = $"{mirror.MirrorDirectoryName}/index.json";
}
if (!sanitized.StartsWith($"{mirror.MirrorDirectoryName}/", StringComparison.OrdinalIgnoreCase))
{
return false;
}
var candidate = Combine(exportDirectory, sanitized);
if (!CandidateWithinExport(exportDirectory, candidate))
{
_logger.LogWarning("Rejected mirror export request for path '{RelativePath}' due to traversal attempt.", relativePath);
return false;
}
if (!File.Exists(candidate))
{
return false;
}
// Extract domain id from path mirror/<domain>/...
var segments = sanitized.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length >= 2)
{
domainId = segments[1];
}
fullPath = candidate;
return true;
}
private bool TryResolveExportDirectory(ConcelierOptions.MirrorOptions mirror, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
{
exportDirectory = null;
exportId = null;
if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute))
{
_logger.LogWarning("Mirror export root is not configured; unable to serve mirror content.");
return false;
}
var root = mirror.ExportRootAbsolute;
var candidateSegment = string.IsNullOrWhiteSpace(mirror.ActiveExportId)
? mirror.LatestDirectoryName
: mirror.ActiveExportId!;
if (TryResolveCandidate(root, candidateSegment, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
{
return true;
}
if (!string.Equals(candidateSegment, mirror.LatestDirectoryName, StringComparison.OrdinalIgnoreCase)
&& TryResolveCandidate(root, mirror.LatestDirectoryName, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
{
return true;
}
try
{
var directories = Directory.Exists(root)
? Directory.GetDirectories(root)
: Array.Empty<string>();
Array.Sort(directories, StringComparer.Ordinal);
Array.Reverse(directories);
foreach (var directory in directories)
{
if (TryResolveCandidate(root, Path.GetFileName(directory), mirror.MirrorDirectoryName, out exportDirectory, out exportId))
{
return true;
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Failed to enumerate export directories under {Root}.", root);
}
return false;
}
private bool TryResolveCandidate(string root, string segment, string mirrorDirectory, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
{
exportDirectory = null;
exportId = null;
if (string.IsNullOrWhiteSpace(segment))
{
return false;
}
var candidate = Path.Combine(root, segment);
if (!Directory.Exists(candidate))
{
return false;
}
var mirrorPath = Path.Combine(candidate, mirrorDirectory);
if (!Directory.Exists(mirrorPath))
{
return false;
}
exportDirectory = candidate;
exportId = segment;
return true;
}
private static string SanitizeRelativePath(string relativePath)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return string.Empty;
}
var trimmed = relativePath.Replace('\\', '/').Trim().TrimStart('/');
return trimmed;
}
private static string Combine(string root, string relativePath)
{
var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length == 0)
{
return Path.GetFullPath(root);
}
var combinedRelative = Path.Combine(segments);
return Path.GetFullPath(Path.Combine(root, combinedRelative));
}
private static bool CandidateWithinExport(string exportDirectory, string candidate)
{
var exportRoot = Path.GetFullPath(exportDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
var candidatePath = Path.GetFullPath(candidate);
return candidatePath.StartsWith(exportRoot, StringComparison.OrdinalIgnoreCase);
}
}
using System;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.WebService.Options;
namespace StellaOps.Concelier.WebService.Services;
internal sealed class MirrorFileLocator
{
private readonly IOptionsMonitor<ConcelierOptions> _options;
private readonly ILogger<MirrorFileLocator> _logger;
public MirrorFileLocator(IOptionsMonitor<ConcelierOptions> options, ILogger<MirrorFileLocator> logger)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public bool TryResolveIndex([NotNullWhen(true)] out string? path, [NotNullWhen(true)] out string? exportId)
=> TryResolveRelativePath("index.json", out path, out exportId, out _);
public bool TryResolveRelativePath(string relativePath, [NotNullWhen(true)] out string? fullPath, [NotNullWhen(true)] out string? exportId, out string? domainId)
{
fullPath = null;
exportId = null;
domainId = null;
var mirror = _options.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirror.Enabled)
{
return false;
}
if (!TryResolveExportDirectory(mirror, out var exportDirectory, out exportId))
{
return false;
}
var sanitized = SanitizeRelativePath(relativePath);
if (sanitized.Length == 0 || string.Equals(sanitized, "index.json", StringComparison.OrdinalIgnoreCase))
{
sanitized = $"{mirror.MirrorDirectoryName}/index.json";
}
if (!sanitized.StartsWith($"{mirror.MirrorDirectoryName}/", StringComparison.OrdinalIgnoreCase))
{
return false;
}
var candidate = Combine(exportDirectory, sanitized);
if (!CandidateWithinExport(exportDirectory, candidate))
{
_logger.LogWarning("Rejected mirror export request for path '{RelativePath}' due to traversal attempt.", relativePath);
return false;
}
if (!File.Exists(candidate))
{
return false;
}
// Extract domain id from path mirror/<domain>/...
var segments = sanitized.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length >= 2)
{
domainId = segments[1];
}
fullPath = candidate;
return true;
}
private bool TryResolveExportDirectory(ConcelierOptions.MirrorOptions mirror, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
{
exportDirectory = null;
exportId = null;
if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute))
{
_logger.LogWarning("Mirror export root is not configured; unable to serve mirror content.");
return false;
}
var root = mirror.ExportRootAbsolute;
var candidateSegment = string.IsNullOrWhiteSpace(mirror.ActiveExportId)
? mirror.LatestDirectoryName
: mirror.ActiveExportId!;
if (TryResolveCandidate(root, candidateSegment, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
{
return true;
}
if (!string.Equals(candidateSegment, mirror.LatestDirectoryName, StringComparison.OrdinalIgnoreCase)
&& TryResolveCandidate(root, mirror.LatestDirectoryName, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
{
return true;
}
try
{
var directories = Directory.Exists(root)
? Directory.GetDirectories(root)
: Array.Empty<string>();
Array.Sort(directories, StringComparer.Ordinal);
Array.Reverse(directories);
foreach (var directory in directories)
{
if (TryResolveCandidate(root, Path.GetFileName(directory), mirror.MirrorDirectoryName, out exportDirectory, out exportId))
{
return true;
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Failed to enumerate export directories under {Root}.", root);
}
return false;
}
private bool TryResolveCandidate(string root, string segment, string mirrorDirectory, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
{
exportDirectory = null;
exportId = null;
if (string.IsNullOrWhiteSpace(segment))
{
return false;
}
var candidate = Path.Combine(root, segment);
if (!Directory.Exists(candidate))
{
return false;
}
var mirrorPath = Path.Combine(candidate, mirrorDirectory);
if (!Directory.Exists(mirrorPath))
{
return false;
}
exportDirectory = candidate;
exportId = segment;
return true;
}
private static string SanitizeRelativePath(string relativePath)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return string.Empty;
}
var trimmed = relativePath.Replace('\\', '/').Trim().TrimStart('/');
return trimmed;
}
private static string Combine(string root, string relativePath)
{
var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length == 0)
{
return Path.GetFullPath(root);
}
var combinedRelative = Path.Combine(segments);
return Path.GetFullPath(Path.Combine(root, combinedRelative));
}
private static bool CandidateWithinExport(string exportDirectory, string candidate)
{
var exportRoot = Path.GetFullPath(exportDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
var candidatePath = Path.GetFullPath(candidate);
return candidatePath.StartsWith(exportRoot, StringComparison.OrdinalIgnoreCase);
}
}

View File

@@ -1,57 +1,57 @@
using Microsoft.Extensions.Caching.Memory;
namespace StellaOps.Concelier.WebService.Services;
internal sealed class MirrorRateLimiter
{
private readonly IMemoryCache _cache;
private readonly TimeProvider _timeProvider;
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
{
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
{
retryAfter = null;
if (limit <= 0 || limit == int.MaxValue)
{
return true;
}
var key = CreateKey(domainId, scope);
var now = _timeProvider.GetUtcNow();
var counter = _cache.Get<Counter>(key);
if (counter is null || now - counter.WindowStart >= Window)
{
counter = new Counter(now, 0);
}
if (counter.Count >= limit)
{
var windowEnd = counter.WindowStart + Window;
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
return false;
}
counter = counter with { Count = counter.Count + 1 };
var absoluteExpiration = counter.WindowStart + Window;
_cache.Set(key, counter, absoluteExpiration);
return true;
}
private static string CreateKey(string domainId, string scope)
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
{
state.domainId.AsSpan().CopyTo(span);
span[state.domainId.Length] = '|';
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
});
private sealed record Counter(DateTimeOffset WindowStart, int Count);
}
using Microsoft.Extensions.Caching.Memory;
namespace StellaOps.Concelier.WebService.Services;
internal sealed class MirrorRateLimiter
{
private readonly IMemoryCache _cache;
private readonly TimeProvider _timeProvider;
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
{
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
{
retryAfter = null;
if (limit <= 0 || limit == int.MaxValue)
{
return true;
}
var key = CreateKey(domainId, scope);
var now = _timeProvider.GetUtcNow();
var counter = _cache.Get<Counter>(key);
if (counter is null || now - counter.WindowStart >= Window)
{
counter = new Counter(now, 0);
}
if (counter.Count >= limit)
{
var windowEnd = counter.WindowStart + Window;
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
return false;
}
counter = counter with { Count = counter.Count + 1 };
var absoluteExpiration = counter.WindowStart + Window;
_cache.Set(key, counter, absoluteExpiration);
return true;
}
private static string CreateKey(string domainId, string scope)
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
{
state.domainId.AsSpan().CopyTo(span);
span[state.domainId.Length] = '|';
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
});
private sealed record Counter(DateTimeOffset WindowStart, int Count);
}

View File

@@ -10,8 +10,8 @@ using System.Xml.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Bson.IO;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Documents.IO;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Fetch;
@@ -292,7 +292,7 @@ public sealed class AcscConnector : IFeedConnector
var dto = AcscFeedParser.Parse(rawBytes, metadata.FeedSlug, parsedAt, _htmlSanitizer);
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = BsonDocument.Parse(json);
var payload = DocumentObject.Parse(json);
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
var dtoRecord = existingDto is null
@@ -678,7 +678,7 @@ public sealed class AcscConnector : IFeedConnector
private Task UpdateCursorAsync(AcscCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "acsc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "acsc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
}
}

View File

@@ -1,44 +1,44 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Acsc.Configuration;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:acsc";
private const string FetchCron = "7,37 * * * *";
private const string ParseCron = "12,42 * * * *";
private const string MapCron = "17,47 * * * *";
private const string ProbeCron = "25,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAcscConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
return services;
}
}
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Acsc.Configuration;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:acsc";
private const string FetchCron = "7,37 * * * *";
private const string ParseCron = "12,42 * * * *";
private const string MapCron = "17,47 * * * *";
private const string ProbeCron = "25,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAcscConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
return services;
}
}

View File

@@ -1,56 +1,56 @@
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.Acsc;
public static class AcscServiceCollectionExtensions
{
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AcscOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = options.UserAgent;
clientOptions.RequestVersion = options.RequestVersion;
clientOptions.VersionPolicy = options.VersionPolicy;
clientOptions.AllowAutoRedirect = true;
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.AllowAutoRedirect = true;
};
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
if (options.RelayEndpoint is not null)
{
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
});
services.AddSingleton<AcscDiagnostics>();
services.AddTransient<AcscConnector>();
return services;
}
}
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.Acsc;
public static class AcscServiceCollectionExtensions
{
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AcscOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = options.UserAgent;
clientOptions.RequestVersion = options.RequestVersion;
clientOptions.VersionPolicy = options.VersionPolicy;
clientOptions.AllowAutoRedirect = true;
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.AllowAutoRedirect = true;
};
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
if (options.RelayEndpoint is not null)
{
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
});
services.AddSingleton<AcscDiagnostics>();
services.AddTransient<AcscConnector>();
return services;
}
}

View File

@@ -1,54 +1,54 @@
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Defines a single ACSC RSS feed endpoint.
/// </summary>
public sealed class AcscFeedOptions
{
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Logical slug for the feed (alerts, advisories, threats, etc.).
/// </summary>
public string Slug { get; set; } = "alerts";
/// <summary>
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
/// </summary>
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
/// <summary>
/// Indicates whether the feed is active.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Optional display name for logging.
/// </summary>
public string? DisplayName { get; set; }
internal void Validate(int index)
{
if (string.IsNullOrWhiteSpace(Slug))
{
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
}
if (!SlugPattern.IsMatch(Slug))
{
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
}
if (string.IsNullOrWhiteSpace(RelativePath))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
}
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
}
}
}
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Defines a single ACSC RSS feed endpoint.
/// </summary>
public sealed class AcscFeedOptions
{
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Logical slug for the feed (alerts, advisories, threats, etc.).
/// </summary>
public string Slug { get; set; } = "alerts";
/// <summary>
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
/// </summary>
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
/// <summary>
/// Indicates whether the feed is active.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Optional display name for logging.
/// </summary>
public string? DisplayName { get; set; }
internal void Validate(int index)
{
if (string.IsNullOrWhiteSpace(Slug))
{
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
}
if (!SlugPattern.IsMatch(Slug))
{
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
}
if (string.IsNullOrWhiteSpace(RelativePath))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
}
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
}
}
}

View File

@@ -1,153 +1,153 @@
using System.Net;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Connector options governing ACSC feed access and retry behaviour.
/// </summary>
public sealed class AcscOptions
{
public const string HttpClientName = "acsc";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
public AcscOptions()
{
Feeds = new List<AcscFeedOptions>
{
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
};
}
/// <summary>
/// Base endpoint for direct ACSC fetches.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
/// <summary>
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
/// </summary>
public Uri? RelayEndpoint { get; set; }
/// <summary>
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
/// </summary>
public bool PreferRelayByDefault { get; set; }
/// <summary>
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
/// </summary>
public bool EnableRelayFallback { get; set; } = true;
/// <summary>
/// If set, the connector will always use the relay endpoint and skip direct attempts.
/// </summary>
public bool ForceRelay { get; set; }
/// <summary>
/// Timeout applied to fetch requests (overrides HttpClient default).
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when marking fetch failures.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Look-back period used when deriving initial published cursors.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
/// <summary>
/// User-agent header sent with outbound requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
/// <summary>
/// RSS feeds requested during fetch.
/// </summary>
public IList<AcscFeedOptions> Feeds { get; }
/// <summary>
/// HTTP version policy requested for outbound requests.
/// </summary>
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
/// <summary>
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
/// </summary>
public Version RequestVersion { get; set; } = HttpVersion.Version20;
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
}
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
}
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
}
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
}
if (Feeds.Count == 0)
{
throw new InvalidOperationException("At least one ACSC feed must be configured.");
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < Feeds.Count; i++)
{
var feed = Feeds[i];
feed.Validate(i);
if (!feed.Enabled)
{
continue;
}
if (!seen.Add(feed.Slug))
{
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
}
}
}
}
using System.Net;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Connector options governing ACSC feed access and retry behaviour.
/// </summary>
public sealed class AcscOptions
{
public const string HttpClientName = "acsc";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
public AcscOptions()
{
Feeds = new List<AcscFeedOptions>
{
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
};
}
/// <summary>
/// Base endpoint for direct ACSC fetches.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
/// <summary>
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
/// </summary>
public Uri? RelayEndpoint { get; set; }
/// <summary>
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
/// </summary>
public bool PreferRelayByDefault { get; set; }
/// <summary>
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
/// </summary>
public bool EnableRelayFallback { get; set; } = true;
/// <summary>
/// If set, the connector will always use the relay endpoint and skip direct attempts.
/// </summary>
public bool ForceRelay { get; set; }
/// <summary>
/// Timeout applied to fetch requests (overrides HttpClient default).
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when marking fetch failures.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Look-back period used when deriving initial published cursors.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
/// <summary>
/// User-agent header sent with outbound requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
/// <summary>
/// RSS feeds requested during fetch.
/// </summary>
public IList<AcscFeedOptions> Feeds { get; }
/// <summary>
/// HTTP version policy requested for outbound requests.
/// </summary>
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
/// <summary>
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
/// </summary>
public Version RequestVersion { get; set; } = HttpVersion.Version20;
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
}
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
}
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
}
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
}
if (Feeds.Count == 0)
{
throw new InvalidOperationException("At least one ACSC feed must be configured.");
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < Feeds.Count; i++)
{
var feed = Feeds[i];
feed.Validate(i);
if (!feed.Enabled)
{
continue;
}
if (!seen.Add(feed.Slug))
{
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
}
}
}
}

View File

@@ -1,141 +1,141 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal enum AcscEndpointPreference
{
Auto = 0,
Direct = 1,
Relay = 2,
}
internal sealed record AcscCursor(
AcscEndpointPreference PreferredEndpoint,
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
public static AcscCursor Empty { get; } = new(
AcscEndpointPreference.Auto,
EmptyFeedDictionary,
EmptyGuidList,
EmptyGuidList);
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
=> this with { PreferredEndpoint = preference };
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
{
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (values is not null)
{
foreach (var kvp in values)
{
snapshot[kvp.Key] = kvp.Value;
}
}
return this with { LastPublishedByFeed = snapshot };
}
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["preferredEndpoint"] = PreferredEndpoint.ToString(),
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
var feedsDocument = new BsonDocument();
foreach (var kvp in LastPublishedByFeed)
{
if (kvp.Value.HasValue)
{
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
}
}
document["feeds"] = feedsDocument;
return document;
}
public static AcscCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
? ParseEndpointPreference(endpointValue.AsString)
: AcscEndpointPreference.Auto;
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is BsonDocument feedsDocument)
{
foreach (var element in feedsDocument.Elements)
{
feeds[element.Name] = ParseDate(element.Value);
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AcscCursor(
preferredEndpoint,
feeds,
pendingDocuments,
pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
private static AcscEndpointPreference ParseEndpointPreference(string? value)
{
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
return AcscEndpointPreference.Auto;
}
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal enum AcscEndpointPreference
{
Auto = 0,
Direct = 1,
Relay = 2,
}
internal sealed record AcscCursor(
AcscEndpointPreference PreferredEndpoint,
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
public static AcscCursor Empty { get; } = new(
AcscEndpointPreference.Auto,
EmptyFeedDictionary,
EmptyGuidList,
EmptyGuidList);
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
=> this with { PreferredEndpoint = preference };
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
{
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (values is not null)
{
foreach (var kvp in values)
{
snapshot[kvp.Key] = kvp.Value;
}
}
return this with { LastPublishedByFeed = snapshot };
}
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["preferredEndpoint"] = PreferredEndpoint.ToString(),
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
var feedsDocument = new DocumentObject();
foreach (var kvp in LastPublishedByFeed)
{
if (kvp.Value.HasValue)
{
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
}
}
document["feeds"] = feedsDocument;
return document;
}
public static AcscCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
? ParseEndpointPreference(endpointValue.AsString)
: AcscEndpointPreference.Auto;
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is DocumentObject feedsDocument)
{
foreach (var element in feedsDocument.Elements)
{
feeds[element.Name] = ParseDate(element.Value);
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AcscCursor(
preferredEndpoint,
feeds,
pendingDocuments,
pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
private static AcscEndpointPreference ParseEndpointPreference(string? value)
{
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
return AcscEndpointPreference.Auto;
}
}

View File

@@ -1,97 +1,97 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
public sealed class AcscDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFallbacks;
private readonly Counter<long> _cursorUpdates;
private readonly Counter<long> _parseAttempts;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
public AcscDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
}
public void FetchAttempt(string feed, string mode)
=> _fetchAttempts.Add(1, GetTags(feed, mode));
public void FetchSuccess(string feed, string mode)
=> _fetchSuccess.Add(1, GetTags(feed, mode));
public void FetchFailure(string feed, string mode)
=> _fetchFailures.Add(1, GetTags(feed, mode));
public void FetchUnchanged(string feed, string mode)
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
public void FetchFallback(string feed, string mode, string reason)
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
public void CursorUpdated(string feed)
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseAttempt(string feed)
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseSuccess(string feed)
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseFailure(string feed, string reason)
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
{
new("feed", feed),
new("reason", reason),
});
public void MapSuccess(int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mapSuccess.Add(advisoryCount);
}
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
};
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
extra,
};
public void Dispose()
{
_meter.Dispose();
}
}
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
public sealed class AcscDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFallbacks;
private readonly Counter<long> _cursorUpdates;
private readonly Counter<long> _parseAttempts;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
public AcscDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
}
public void FetchAttempt(string feed, string mode)
=> _fetchAttempts.Add(1, GetTags(feed, mode));
public void FetchSuccess(string feed, string mode)
=> _fetchSuccess.Add(1, GetTags(feed, mode));
public void FetchFailure(string feed, string mode)
=> _fetchFailures.Add(1, GetTags(feed, mode));
public void FetchUnchanged(string feed, string mode)
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
public void FetchFallback(string feed, string mode, string reason)
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
public void CursorUpdated(string feed)
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseAttempt(string feed)
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseSuccess(string feed)
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseFailure(string feed, string reason)
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
{
new("feed", feed),
new("reason", reason),
});
public void MapSuccess(int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mapSuccess.Add(advisoryCount);
}
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
};
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
extra,
};
public void Dispose()
{
_meter.Dispose();
}
}

View File

@@ -1,20 +1,20 @@
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
{
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
{
if (document.Metadata is null)
{
return new AcscDocumentMetadata(string.Empty, string.Empty);
}
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
return new AcscDocumentMetadata(
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
}
}
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
{
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
{
if (document.Metadata is null)
{
return new AcscDocumentMetadata(string.Empty, string.Empty);
}
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
return new AcscDocumentMetadata(
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
}
}

View File

@@ -1,58 +1,58 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal sealed record AcscFeedDto(
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
[property: JsonPropertyName("feedLink")] string? FeedLink,
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
{
public static AcscFeedDto Empty { get; } = new(
FeedSlug: string.Empty,
FeedTitle: null,
FeedLink: null,
FeedUpdated: null,
ParsedAt: DateTimeOffset.UnixEpoch,
Entries: Array.Empty<AcscEntryDto>());
}
internal sealed record AcscEntryDto(
[property: JsonPropertyName("entryId")] string EntryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("link")] string? Link,
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("published")] DateTimeOffset? Published,
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
[property: JsonPropertyName("summary")] string Summary,
[property: JsonPropertyName("contentHtml")] string ContentHtml,
[property: JsonPropertyName("contentText")] string ContentText,
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
{
public static AcscEntryDto Empty { get; } = new(
EntryId: string.Empty,
Title: string.Empty,
Link: null,
FeedSlug: string.Empty,
Published: null,
Updated: null,
Summary: string.Empty,
ContentHtml: string.Empty,
ContentText: string.Empty,
References: Array.Empty<AcscReferenceDto>(),
Aliases: Array.Empty<string>(),
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
internal sealed record AcscReferenceDto(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url)
{
public static AcscReferenceDto Empty { get; } = new(
Title: string.Empty,
Url: string.Empty);
}
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal sealed record AcscFeedDto(
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
[property: JsonPropertyName("feedLink")] string? FeedLink,
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
{
public static AcscFeedDto Empty { get; } = new(
FeedSlug: string.Empty,
FeedTitle: null,
FeedLink: null,
FeedUpdated: null,
ParsedAt: DateTimeOffset.UnixEpoch,
Entries: Array.Empty<AcscEntryDto>());
}
internal sealed record AcscEntryDto(
[property: JsonPropertyName("entryId")] string EntryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("link")] string? Link,
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("published")] DateTimeOffset? Published,
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
[property: JsonPropertyName("summary")] string Summary,
[property: JsonPropertyName("contentHtml")] string ContentHtml,
[property: JsonPropertyName("contentText")] string ContentText,
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
{
public static AcscEntryDto Empty { get; } = new(
EntryId: string.Empty,
Title: string.Empty,
Link: null,
FeedSlug: string.Empty,
Published: null,
Updated: null,
Summary: string.Empty,
ContentHtml: string.Empty,
ContentText: string.Empty,
References: Array.Empty<AcscReferenceDto>(),
Aliases: Array.Empty<string>(),
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
internal sealed record AcscReferenceDto(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url)
{
public static AcscReferenceDto Empty { get; } = new(
Title: string.Empty,
Url: string.Empty);
}

View File

@@ -1,312 +1,312 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal static class AcscMapper
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<Advisory> Map(
AcscFeedDto feed,
DocumentRecord document,
DtoRecord dtoRecord,
string sourceName,
DateTimeOffset mappedAt)
{
ArgumentNullException.ThrowIfNull(feed);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
if (feed.Entries is null || feed.Entries.Count == 0)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(feed.Entries.Count);
foreach (var entry in feed.Entries)
{
if (entry is null)
{
continue;
}
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
var fetchProvenance = new AdvisoryProvenance(
sourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var feedProvenance = new AdvisoryProvenance(
sourceName,
"feed",
feed.FeedSlug ?? string.Empty,
feed.ParsedAt.ToUniversalTime(),
fieldMask: new[] { "summary" });
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
var provenance = new[]
{
fetchProvenance,
feedProvenance,
mappingProvenance,
};
var aliases = BuildAliases(entry);
var severity = TryGetSeverity(entry.Fields);
var references = BuildReferences(entry, sourceName, mappedAt);
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
var advisory = new Advisory(
advisoryKey,
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
language: "en",
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
modified: entry.Updated?.ToUniversalTime(),
severity: severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
advisories.Add(advisory);
}
return advisories;
}
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(entry.EntryId))
{
aliases.Add(entry.EntryId.Trim());
}
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
{
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias.Trim());
}
}
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
return aliases.Count == 0
? Array.Empty<string>()
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Validation.LooksLikeHttpUrl(url))
{
return;
}
if (!seen.Add(url))
{
return;
}
references.Add(new AdvisoryReference(
url,
kind,
sourceTag,
summary,
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
}
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
{
if (reference is null)
{
continue;
}
AddReference(reference.Url, "reference", null, reference.Title);
}
return references.Count == 0
? Array.Empty<AdvisoryReference>()
: references
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
if (entry.Fields is null || entry.Fields.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
if (string.IsNullOrWhiteSpace(systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
var identifiers = systemsAffected
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(static value => value.Trim())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (identifiers.Length == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(identifiers.Length);
foreach (var identifier in identifiers)
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
{
if (fields is null || fields.Count == 0)
{
return null;
}
var keys = new[]
{
"severity",
"riskLevel",
"threatLevel",
"impact",
};
foreach (var key in keys)
{
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
{
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
? entry.EntryId
: !string.IsNullOrWhiteSpace(entry.Link)
? entry.Link
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
}
return $"{sourceName}/{slug}/{identifier}";
}
private static string ToSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var builder = new StringBuilder(value.Length);
var previousDash = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
previousDash = false;
}
else if (!previousDash)
{
builder.Append('-');
previousDash = true;
}
}
var slug = builder.ToString().Trim('-');
if (string.IsNullOrEmpty(slug))
{
slug = CreateHash(value);
}
return slug.Length <= 64 ? slug : slug[..64];
}
private static string CreateHash(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
}
}
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal static class AcscMapper
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<Advisory> Map(
AcscFeedDto feed,
DocumentRecord document,
DtoRecord dtoRecord,
string sourceName,
DateTimeOffset mappedAt)
{
ArgumentNullException.ThrowIfNull(feed);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
if (feed.Entries is null || feed.Entries.Count == 0)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(feed.Entries.Count);
foreach (var entry in feed.Entries)
{
if (entry is null)
{
continue;
}
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
var fetchProvenance = new AdvisoryProvenance(
sourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var feedProvenance = new AdvisoryProvenance(
sourceName,
"feed",
feed.FeedSlug ?? string.Empty,
feed.ParsedAt.ToUniversalTime(),
fieldMask: new[] { "summary" });
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
var provenance = new[]
{
fetchProvenance,
feedProvenance,
mappingProvenance,
};
var aliases = BuildAliases(entry);
var severity = TryGetSeverity(entry.Fields);
var references = BuildReferences(entry, sourceName, mappedAt);
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
var advisory = new Advisory(
advisoryKey,
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
language: "en",
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
modified: entry.Updated?.ToUniversalTime(),
severity: severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
advisories.Add(advisory);
}
return advisories;
}
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(entry.EntryId))
{
aliases.Add(entry.EntryId.Trim());
}
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
{
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias.Trim());
}
}
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
return aliases.Count == 0
? Array.Empty<string>()
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Validation.LooksLikeHttpUrl(url))
{
return;
}
if (!seen.Add(url))
{
return;
}
references.Add(new AdvisoryReference(
url,
kind,
sourceTag,
summary,
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
}
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
{
if (reference is null)
{
continue;
}
AddReference(reference.Url, "reference", null, reference.Title);
}
return references.Count == 0
? Array.Empty<AdvisoryReference>()
: references
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
if (entry.Fields is null || entry.Fields.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
if (string.IsNullOrWhiteSpace(systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
var identifiers = systemsAffected
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(static value => value.Trim())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (identifiers.Length == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(identifiers.Length);
foreach (var identifier in identifiers)
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
{
if (fields is null || fields.Count == 0)
{
return null;
}
var keys = new[]
{
"severity",
"riskLevel",
"threatLevel",
"impact",
};
foreach (var key in keys)
{
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
{
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
? entry.EntryId
: !string.IsNullOrWhiteSpace(entry.Link)
? entry.Link
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
}
return $"{sourceName}/{slug}/{identifier}";
}
private static string ToSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var builder = new StringBuilder(value.Length);
var previousDash = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
previousDash = false;
}
else if (!previousDash)
{
builder.Append('-');
previousDash = true;
}
}
var slug = builder.ToString().Trim('-');
if (string.IsNullOrEmpty(slug))
{
slug = CreateHash(value);
}
return slug.Length <= 64 ? slug : slug[..64];
}
private static string CreateHash(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
}
}

View File

@@ -1,55 +1,55 @@
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Acsc;
internal static class AcscJobKinds
{
public const string Fetch = "source:acsc:fetch";
public const string Parse = "source:acsc:parse";
public const string Map = "source:acsc:map";
public const string Probe = "source:acsc:probe";
}
internal sealed class AcscFetchJob : IJob
{
private readonly AcscConnector _connector;
public AcscFetchJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AcscParseJob : IJob
{
private readonly AcscConnector _connector;
public AcscParseJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AcscMapJob : IJob
{
private readonly AcscConnector _connector;
public AcscMapJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
internal sealed class AcscProbeJob : IJob
{
private readonly AcscConnector _connector;
public AcscProbeJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ProbeAsync(cancellationToken);
}
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Acsc;
internal static class AcscJobKinds
{
public const string Fetch = "source:acsc:fetch";
public const string Parse = "source:acsc:parse";
public const string Map = "source:acsc:map";
public const string Probe = "source:acsc:probe";
}
internal sealed class AcscFetchJob : IJob
{
private readonly AcscConnector _connector;
public AcscFetchJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AcscParseJob : IJob
{
private readonly AcscConnector _connector;
public AcscParseJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AcscMapJob : IJob
{
private readonly AcscConnector _connector;
public AcscMapJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
internal sealed class AcscProbeJob : IJob
{
private readonly AcscConnector _connector;
public AcscProbeJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ProbeAsync(cancellationToken);
}

View File

@@ -1,4 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]

View File

@@ -10,7 +10,7 @@ using System.Threading.Tasks;
using System.Globalization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Cccs.Internal;
using StellaOps.Concelier.Connector.Common;
@@ -332,7 +332,7 @@ public sealed class CccsConnector : IFeedConnector
}
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var dtoBson = BsonDocument.Parse(dtoJson);
var dtoBson = DocumentObject.Parse(dtoJson);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -464,7 +464,7 @@ public sealed class CccsConnector : IFeedConnector
private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cccs";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CccsConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CccsConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cccs";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CccsConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CccsConnector>();
}
}

View File

@@ -1,50 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Cccs.Configuration;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cccs";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCccsConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CccsFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Cccs.Configuration;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cccs";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCccsConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CccsFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -1,47 +1,47 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Cccs.Internal;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs;
public static class CccsServiceCollectionExtensions
{
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CccsOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0";
clientOptions.Timeout = options.RequestTimeout;
clientOptions.AllowedHosts.Clear();
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
{
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
}
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
clientOptions.AllowedHosts.Add("cyber.gc.ca");
});
services.TryAddSingleton<HtmlContentSanitizer>();
services.TryAddSingleton<CccsDiagnostics>();
services.TryAddSingleton<CccsHtmlParser>();
services.TryAddSingleton<CccsFeedClient>();
services.AddTransient<CccsConnector>();
return services;
}
}
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Cccs.Internal;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs;
public static class CccsServiceCollectionExtensions
{
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CccsOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0";
clientOptions.Timeout = options.RequestTimeout;
clientOptions.AllowedHosts.Clear();
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
{
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
}
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
clientOptions.AllowedHosts.Add("cyber.gc.ca");
});
services.TryAddSingleton<HtmlContentSanitizer>();
services.TryAddSingleton<CccsDiagnostics>();
services.TryAddSingleton<CccsHtmlParser>();
services.TryAddSingleton<CccsFeedClient>();
services.AddTransient<CccsConnector>();
return services;
}
}

View File

@@ -1,130 +1,130 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Cccs.Configuration;
public sealed class CccsOptions
{
public const string HttpClientName = "concelier.source.cccs";
private readonly List<CccsFeedEndpoint> _feeds = new();
public CccsOptions()
{
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
}
/// <summary>
/// Feed endpoints to poll; configure per language or content category.
/// </summary>
public IList<CccsFeedEndpoint> Feeds => _feeds;
/// <summary>
/// Maximum number of entries to enqueue per fetch cycle.
/// </summary>
public int MaxEntriesPerFetch { get; set; } = 80;
/// <summary>
/// Maximum remembered entries (URI+hash) for deduplication.
/// </summary>
public int MaxKnownEntries { get; set; } = 512;
/// <summary>
/// Timeout applied to feed and taxonomy requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay between successive feed requests to respect upstream throttling.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when fetch fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
public void Validate()
{
if (_feeds.Count == 0)
{
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
}
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var feed in _feeds)
{
feed.Validate();
if (!seenLanguages.Add(feed.Language))
{
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
}
}
if (MaxEntriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
}
if (MaxKnownEntries <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}
public sealed class CccsFeedEndpoint
{
public CccsFeedEndpoint()
{
}
public CccsFeedEndpoint(string language, Uri uri)
{
Language = language;
Uri = uri;
}
public string Language { get; set; } = "en";
public Uri? Uri { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Language))
{
throw new InvalidOperationException("CCCS feed language must be specified.");
}
if (Uri is null || !Uri.IsAbsoluteUri)
{
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
}
}
public Uri BuildTaxonomyUri()
{
if (Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
}
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Cccs.Configuration;
public sealed class CccsOptions
{
public const string HttpClientName = "concelier.source.cccs";
private readonly List<CccsFeedEndpoint> _feeds = new();
public CccsOptions()
{
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
}
/// <summary>
/// Feed endpoints to poll; configure per language or content category.
/// </summary>
public IList<CccsFeedEndpoint> Feeds => _feeds;
/// <summary>
/// Maximum number of entries to enqueue per fetch cycle.
/// </summary>
public int MaxEntriesPerFetch { get; set; } = 80;
/// <summary>
/// Maximum remembered entries (URI+hash) for deduplication.
/// </summary>
public int MaxKnownEntries { get; set; } = 512;
/// <summary>
/// Timeout applied to feed and taxonomy requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay between successive feed requests to respect upstream throttling.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when fetch fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
public void Validate()
{
if (_feeds.Count == 0)
{
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
}
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var feed in _feeds)
{
feed.Validate();
if (!seenLanguages.Add(feed.Language))
{
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
}
}
if (MaxEntriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
}
if (MaxKnownEntries <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}
public sealed class CccsFeedEndpoint
{
public CccsFeedEndpoint()
{
}
public CccsFeedEndpoint(string language, Uri uri)
{
Language = language;
Uri = uri;
}
public string Language { get; set; } = "en";
public Uri? Uri { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Language))
{
throw new InvalidOperationException("CCCS feed language must be specified.");
}
if (Uri is null || !Uri.IsAbsoluteUri)
{
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
}
}
public Uri BuildTaxonomyUri()
{
if (Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
}
var language = Uri.GetQueryParameterValueOrDefault("lang", Language);
var taxonomyBuilder = new UriBuilder(Uri)
{
@@ -135,46 +135,46 @@ public sealed class CccsFeedEndpoint
return taxonomyBuilder.Uri;
}
}
internal static class CccsUriExtensions
{
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
{
if (uri is null)
{
return fallback;
}
var query = uri.Query;
if (string.IsNullOrEmpty(query))
{
return fallback;
}
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex < 0)
{
continue;
}
var left = pair[..separatorIndex].Trim();
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var right = pair[(separatorIndex + 1)..].Trim();
if (right.Length == 0)
{
continue;
}
return Uri.UnescapeDataString(right);
}
return fallback;
}
}
internal static class CccsUriExtensions
{
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
{
if (uri is null)
{
return fallback;
}
var query = uri.Query;
if (string.IsNullOrEmpty(query))
{
return fallback;
}
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex < 0)
{
continue;
}
var left = pair[..separatorIndex].Trim();
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var right = pair[(separatorIndex + 1)..].Trim();
if (right.Length == 0)
{
continue;
}
return Uri.UnescapeDataString(right);
}
return fallback;
}
}

View File

@@ -1,54 +1,54 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsAdvisoryDto
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string SerialNumber { get; init; } = string.Empty;
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("products")]
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
[JsonPropertyName("references")]
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
}
internal sealed record CccsReferenceDto(
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("label")] string? Label);
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsAdvisoryDto
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string SerialNumber { get; init; } = string.Empty;
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("products")]
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
[JsonPropertyName("references")]
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
}
internal sealed record CccsReferenceDto(
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("label")] string? Label);

View File

@@ -1,145 +1,145 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, string> KnownEntryHashes,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingMappings = distinct };
}
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
{
var map = hashes is null || hashes.Count == 0
? EmptyHashes
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
return this with { KnownEntryHashes = map };
}
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var doc = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (KnownEntryHashes.Count > 0)
{
var hashes = new BsonArray();
foreach (var kvp in KnownEntryHashes)
{
hashes.Add(new BsonDocument
{
["uri"] = kvp.Key,
["hash"] = kvp.Value,
});
}
doc["knownEntryHashes"] = hashes;
}
if (LastFetchAt.HasValue)
{
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return doc;
}
public static CccsCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var hashes = ReadHashMap(document);
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
? ParseDateTime(value)
: null;
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidCollection;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
items.Add(guid);
}
}
return items;
}
private static IReadOnlyDictionary<string, string> ReadHashMap(BsonDocument document)
{
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0)
{
return EmptyHashes;
}
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
foreach (var element in array)
{
if (element is not BsonDocument entry)
{
continue;
}
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString))
{
continue;
}
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull
? hashValue.AsString
: string.Empty;
map[uriValue.AsString] = hash;
}
return map;
}
private static DateTimeOffset? ParseDateTime(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, string> KnownEntryHashes,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingMappings = distinct };
}
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
{
var map = hashes is null || hashes.Count == 0
? EmptyHashes
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
return this with { KnownEntryHashes = map };
}
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public DocumentObject ToDocumentObject()
{
var doc = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (KnownEntryHashes.Count > 0)
{
var hashes = new DocumentArray();
foreach (var kvp in KnownEntryHashes)
{
hashes.Add(new DocumentObject
{
["uri"] = kvp.Key,
["hash"] = kvp.Value,
});
}
doc["knownEntryHashes"] = hashes;
}
if (LastFetchAt.HasValue)
{
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return doc;
}
public static CccsCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var hashes = ReadHashMap(document);
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
? ParseDateTime(value)
: null;
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidCollection;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
items.Add(guid);
}
}
return items;
}
private static IReadOnlyDictionary<string, string> ReadHashMap(DocumentObject document)
{
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyHashes;
}
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
foreach (var element in array)
{
if (element is not DocumentObject entry)
{
continue;
}
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsDocumentNull || string.IsNullOrWhiteSpace(uriValue.AsString))
{
continue;
}
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsDocumentNull
? hashValue.AsString
: string.Empty;
map[uriValue.AsString] = hash;
}
return map;
}
private static DateTimeOffset? ParseDateTime(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -1,58 +1,58 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Cccs";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CccsDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchSuccess() => _fetchSuccess.Add(1);
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void ParseQuarantine() => _parseQuarantine.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void Dispose() => _meter.Dispose();
}
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Cccs";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CccsDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchSuccess() => _fetchSuccess.Add(1);
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void ParseQuarantine() => _parseQuarantine.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void Dispose() => _meter.Dispose();
}

View File

@@ -1,146 +1,146 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Common.Fetch;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsFeedClient
{
private static readonly string[] AcceptHeaders =
{
"application/json",
"application/vnd.api+json;q=0.9",
"text/json;q=0.8",
"application/*+json;q=0.7",
};
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(endpoint);
if (endpoint.Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured.");
}
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = requestTimeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.feedUri"] = endpoint.Uri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
return CccsFeedResult.Empty;
}
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
if (feedResponse is null || feedResponse.Error)
{
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
return CccsFeedResult.Empty;
}
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
return new CccsFeedResult(items, taxonomy, result.LastModified);
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
throw;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
throw;
}
}
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
{
var taxonomyUri = endpoint.BuildTaxonomyUri();
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = timeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
foreach (var item in taxonomyResponse.Response)
{
if (!string.IsNullOrWhiteSpace(item.Title))
{
map[item.Id] = item.Title!;
}
}
return map;
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
}
private static T? Deserialize<T>(byte[] content)
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
}
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Common.Fetch;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsFeedClient
{
private static readonly string[] AcceptHeaders =
{
"application/json",
"application/vnd.api+json;q=0.9",
"text/json;q=0.8",
"application/*+json;q=0.7",
};
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(endpoint);
if (endpoint.Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured.");
}
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = requestTimeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.feedUri"] = endpoint.Uri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
return CccsFeedResult.Empty;
}
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
if (feedResponse is null || feedResponse.Error)
{
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
return CccsFeedResult.Empty;
}
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
return new CccsFeedResult(items, taxonomy, result.LastModified);
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
throw;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
throw;
}
}
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
{
var taxonomyUri = endpoint.BuildTaxonomyUri();
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = timeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
foreach (var item in taxonomyResponse.Response)
{
if (!string.IsNullOrWhiteSpace(item.Title))
{
map[item.Id] = item.Title!;
}
}
return map;
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
}
private static T? Deserialize<T>(byte[] content)
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
}

View File

@@ -1,101 +1,101 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed class CccsFeedResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsFeedItem> Response { get; init; } = new();
}
internal sealed class CccsFeedItem
{
[JsonPropertyName("nid")]
public int Nid { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("lang")]
public string? Language { get; init; }
[JsonPropertyName("date_modified")]
public string? DateModified { get; init; }
[JsonPropertyName("date_modified_ts")]
public string? DateModifiedTimestamp { get; init; }
[JsonPropertyName("date_created")]
public string? DateCreated { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("body")]
public string[] Body { get; init; } = Array.Empty<string>();
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("alert_type")]
public JsonElement AlertType { get; init; }
[JsonPropertyName("serial_number")]
public string? SerialNumber { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("moderation_state")]
public string? ModerationState { get; init; }
[JsonPropertyName("external_url")]
public string? ExternalUrl { get; init; }
}
internal sealed class CccsTaxonomyResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsTaxonomyItem> Response { get; init; } = new();
}
internal sealed class CccsTaxonomyItem
{
[JsonPropertyName("id")]
public int Id { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
}
internal sealed record CccsFeedResult(
IReadOnlyList<CccsFeedItem> Items,
IReadOnlyDictionary<int, string> AlertTypes,
DateTimeOffset? LastModifiedUtc)
{
public static CccsFeedResult Empty { get; } = new(
Array.Empty<CccsFeedItem>(),
new Dictionary<int, string>(0),
null);
}
internal static class CccsFeedResultExtensions
{
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
=> new(items, alertTypes, lastModified);
}
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed class CccsFeedResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsFeedItem> Response { get; init; } = new();
}
internal sealed class CccsFeedItem
{
[JsonPropertyName("nid")]
public int Nid { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("lang")]
public string? Language { get; init; }
[JsonPropertyName("date_modified")]
public string? DateModified { get; init; }
[JsonPropertyName("date_modified_ts")]
public string? DateModifiedTimestamp { get; init; }
[JsonPropertyName("date_created")]
public string? DateCreated { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("body")]
public string[] Body { get; init; } = Array.Empty<string>();
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("alert_type")]
public JsonElement AlertType { get; init; }
[JsonPropertyName("serial_number")]
public string? SerialNumber { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("moderation_state")]
public string? ModerationState { get; init; }
[JsonPropertyName("external_url")]
public string? ExternalUrl { get; init; }
}
internal sealed class CccsTaxonomyResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsTaxonomyItem> Response { get; init; } = new();
}
internal sealed class CccsTaxonomyItem
{
[JsonPropertyName("id")]
public int Id { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
}
internal sealed record CccsFeedResult(
IReadOnlyList<CccsFeedItem> Items,
IReadOnlyDictionary<int, string> AlertTypes,
DateTimeOffset? LastModifiedUtc)
{
public static CccsFeedResult Empty { get; } = new(
Array.Empty<CccsFeedItem>(),
new Dictionary<int, string>(0),
null);
}
internal static class CccsFeedResultExtensions
{
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
=> new(items, alertTypes, lastModified);
}

View File

@@ -1,353 +1,353 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
private static readonly CultureInfo[] EnglishCultures =
{
CultureInfo.GetCultureInfo("en-CA"),
CultureInfo.GetCultureInfo("en-US"),
CultureInfo.InvariantCulture,
};
private static readonly CultureInfo[] FrenchCultures =
{
CultureInfo.GetCultureInfo("fr-CA"),
CultureInfo.GetCultureInfo("fr-FR"),
CultureInfo.InvariantCulture,
};
private static readonly string[] ProductHeadingKeywords =
{
"affected",
"produit",
"produits",
"produits touch",
"produits concern",
"mesures recommand",
};
private static readonly string[] TrackingParameterPrefixes =
{
"utm_",
"mc_",
"mkt_",
"elq",
};
private readonly HtmlContentSanitizer _sanitizer;
private readonly HtmlParser _parser;
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
{
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
_parser = new HtmlParser(new HtmlParserOptions
{
IsScripting = false,
IsKeepingSourceReferences = false,
});
}
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
{
ArgumentNullException.ThrowIfNull(raw);
var baseUri = TryCreateUri(raw.CanonicalUrl);
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
var body = document.Body ?? document.DocumentElement;
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
var contentRoot = body ?? document.DocumentElement;
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
? raw.SerialNumber!.Trim()
: ExtractSerialNumber(document) ?? raw.SourceId;
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
var products = ExtractProducts(contentRoot);
var cveIds = ExtractCveIds(document);
return new CccsAdvisoryDto
{
SourceId = raw.SourceId,
SerialNumber = serialNumber,
Language = raw.Language,
Title = raw.Title,
Summary = CollapseWhitespace(raw.Summary),
CanonicalUrl = raw.CanonicalUrl,
ContentHtml = sanitized,
Published = published,
Modified = raw.Modified ?? published,
AlertType = raw.AlertType,
Subject = raw.Subject,
Products = products,
References = references,
CveIds = cveIds,
};
}
private static Uri? TryCreateUri(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
}
private static string? ExtractSerialNumber(IDocument document)
{
if (document.Body is null)
{
return null;
}
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = SerialRegex.Match(text);
if (match.Success && match.Groups["id"].Success)
{
var value = match.Groups["id"].Value.Trim();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
var bodyText = document.Body.TextContent;
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
return fallback.Success && fallback.Groups["id"].Success
? fallback.Groups["id"].Value.Trim()
: null;
}
private static DateTimeOffset? ExtractDate(IDocument document, string language)
{
if (document.Body is null)
{
return null;
}
var textSegments = new List<string>();
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = DateRegex.Match(text);
if (match.Success && match.Groups["date"].Success)
{
textSegments.Add(match.Groups["date"].Value.Trim());
}
}
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
{
textSegments.Add(document.Body.TextContent);
}
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
foreach (var segment in textSegments)
{
foreach (var culture in cultures)
{
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return new DateTimeOffset(parsed.ToUniversalTime());
}
}
}
return null;
}
private static IReadOnlyList<string> ExtractProducts(IElement? root)
{
if (root is null)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
{
var text = heading.TextContent?.Trim();
if (!IsProductHeading(text))
{
continue;
}
var sibling = heading.NextElementSibling;
while (sibling is not null)
{
if (IsHeading(sibling))
{
break;
}
if (IsListElement(sibling))
{
AppendListItems(sibling, results);
if (results.Count > 0)
{
break;
}
}
else if (IsContentContainer(sibling))
{
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
{
AppendListItems(list, results);
}
if (results.Count > 0)
{
break;
}
}
sibling = sibling.NextElementSibling;
}
if (results.Count > 0)
{
break;
}
}
if (results.Count == 0)
{
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
{
var itemText = CollapseWhitespace(li.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
results.Add(itemText);
}
}
}
return results.Count == 0
? Array.Empty<string>()
: results
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static bool IsProductHeading(string? heading)
{
if (string.IsNullOrWhiteSpace(heading))
{
return false;
}
var lowered = heading.ToLowerInvariant();
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
}
private static bool IsHeading(IElement element)
=> element.LocalName.Length == 2
&& element.LocalName[0] == 'h'
&& char.IsDigit(element.LocalName[1]);
private static bool IsListElement(IElement element)
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
private static bool IsContentContainer(IElement element)
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
{
foreach (var li in listElement.QuerySelectorAll("li"))
{
if (li is null)
{
continue;
}
var clone = li.Clone(true) as IElement;
if (clone is null)
{
continue;
}
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
{
nested.Remove();
}
var itemText = CollapseWhitespace(clone.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
buffer.Add(itemText);
}
}
}
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
{
if (root is null)
{
return Array.Empty<CccsReferenceDto>();
}
var references = new List<CccsReferenceDto>();
foreach (var anchor in root.QuerySelectorAll("a[href]"))
{
var href = anchor.GetAttribute("href");
var normalized = NormalizeReferenceUrl(href, baseUri, language);
if (normalized is null)
{
continue;
}
var label = CollapseWhitespace(anchor.TextContent);
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
}
return references.Count == 0
? Array.Empty<CccsReferenceDto>()
: references
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
{
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
private static readonly CultureInfo[] EnglishCultures =
{
CultureInfo.GetCultureInfo("en-CA"),
CultureInfo.GetCultureInfo("en-US"),
CultureInfo.InvariantCulture,
};
private static readonly CultureInfo[] FrenchCultures =
{
CultureInfo.GetCultureInfo("fr-CA"),
CultureInfo.GetCultureInfo("fr-FR"),
CultureInfo.InvariantCulture,
};
private static readonly string[] ProductHeadingKeywords =
{
"affected",
"produit",
"produits",
"produits touch",
"produits concern",
"mesures recommand",
};
private static readonly string[] TrackingParameterPrefixes =
{
"utm_",
"mc_",
"mkt_",
"elq",
};
private readonly HtmlContentSanitizer _sanitizer;
private readonly HtmlParser _parser;
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
{
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
_parser = new HtmlParser(new HtmlParserOptions
{
IsScripting = false,
IsKeepingSourceReferences = false,
});
}
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
{
ArgumentNullException.ThrowIfNull(raw);
var baseUri = TryCreateUri(raw.CanonicalUrl);
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
var body = document.Body ?? document.DocumentElement;
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
var contentRoot = body ?? document.DocumentElement;
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
? raw.SerialNumber!.Trim()
: ExtractSerialNumber(document) ?? raw.SourceId;
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
var products = ExtractProducts(contentRoot);
var cveIds = ExtractCveIds(document);
return new CccsAdvisoryDto
{
SourceId = raw.SourceId,
SerialNumber = serialNumber,
Language = raw.Language,
Title = raw.Title,
Summary = CollapseWhitespace(raw.Summary),
CanonicalUrl = raw.CanonicalUrl,
ContentHtml = sanitized,
Published = published,
Modified = raw.Modified ?? published,
AlertType = raw.AlertType,
Subject = raw.Subject,
Products = products,
References = references,
CveIds = cveIds,
};
}
private static Uri? TryCreateUri(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
}
private static string? ExtractSerialNumber(IDocument document)
{
if (document.Body is null)
{
return null;
}
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = SerialRegex.Match(text);
if (match.Success && match.Groups["id"].Success)
{
var value = match.Groups["id"].Value.Trim();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
var bodyText = document.Body.TextContent;
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
return fallback.Success && fallback.Groups["id"].Success
? fallback.Groups["id"].Value.Trim()
: null;
}
private static DateTimeOffset? ExtractDate(IDocument document, string language)
{
if (document.Body is null)
{
return null;
}
var textSegments = new List<string>();
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = DateRegex.Match(text);
if (match.Success && match.Groups["date"].Success)
{
textSegments.Add(match.Groups["date"].Value.Trim());
}
}
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
{
textSegments.Add(document.Body.TextContent);
}
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
foreach (var segment in textSegments)
{
foreach (var culture in cultures)
{
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return new DateTimeOffset(parsed.ToUniversalTime());
}
}
}
return null;
}
private static IReadOnlyList<string> ExtractProducts(IElement? root)
{
if (root is null)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
{
var text = heading.TextContent?.Trim();
if (!IsProductHeading(text))
{
continue;
}
var sibling = heading.NextElementSibling;
while (sibling is not null)
{
if (IsHeading(sibling))
{
break;
}
if (IsListElement(sibling))
{
AppendListItems(sibling, results);
if (results.Count > 0)
{
break;
}
}
else if (IsContentContainer(sibling))
{
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
{
AppendListItems(list, results);
}
if (results.Count > 0)
{
break;
}
}
sibling = sibling.NextElementSibling;
}
if (results.Count > 0)
{
break;
}
}
if (results.Count == 0)
{
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
{
var itemText = CollapseWhitespace(li.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
results.Add(itemText);
}
}
}
return results.Count == 0
? Array.Empty<string>()
: results
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static bool IsProductHeading(string? heading)
{
if (string.IsNullOrWhiteSpace(heading))
{
return false;
}
var lowered = heading.ToLowerInvariant();
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
}
private static bool IsHeading(IElement element)
=> element.LocalName.Length == 2
&& element.LocalName[0] == 'h'
&& char.IsDigit(element.LocalName[1]);
private static bool IsListElement(IElement element)
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
private static bool IsContentContainer(IElement element)
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
{
foreach (var li in listElement.QuerySelectorAll("li"))
{
if (li is null)
{
continue;
}
var clone = li.Clone(true) as IElement;
if (clone is null)
{
continue;
}
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
{
nested.Remove();
}
var itemText = CollapseWhitespace(clone.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
buffer.Add(itemText);
}
}
}
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
{
if (root is null)
{
return Array.Empty<CccsReferenceDto>();
}
var references = new List<CccsReferenceDto>();
foreach (var anchor in root.QuerySelectorAll("a[href]"))
{
var href = anchor.GetAttribute("href");
var normalized = NormalizeReferenceUrl(href, baseUri, language);
if (normalized is null)
{
continue;
}
var label = CollapseWhitespace(anchor.TextContent);
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
}
return references.Count == 0
? Array.Empty<CccsReferenceDto>()
: references
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
{
if (string.IsNullOrWhiteSpace(href))
{
return null;
@@ -363,89 +363,89 @@ public sealed class CccsHtmlParser
}
}
var builder = new UriBuilder(absolute)
{
Fragment = string.Empty,
};
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
builder.Query = filteredQuery;
return builder.Uri.ToString();
}
private static string FilterTrackingParameters(string query, Uri uri, string language)
{
if (string.IsNullOrWhiteSpace(query))
{
return string.Empty;
}
var trimmed = query.TrimStart('?');
if (string.IsNullOrWhiteSpace(trimmed))
{
return string.Empty;
}
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
var kept = new List<string>();
foreach (var parameter in parameters)
{
var separatorIndex = parameter.IndexOf('=');
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
{
kept.Add($"lang={language}");
continue;
}
kept.Add(parameter);
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
{
kept.Add($"lang={language}");
}
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
}
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
{
if (document.Body is null)
{
return Array.Empty<string>();
}
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
if (matches.Count == 0)
{
return Array.Empty<string>();
}
return matches
.Select(match => match.Value.ToUpperInvariant())
.Distinct(StringComparer.Ordinal)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
}
private static string? CollapseWhitespace(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
return collapsed.Length == 0 ? null : collapsed;
}
}
var builder = new UriBuilder(absolute)
{
Fragment = string.Empty,
};
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
builder.Query = filteredQuery;
return builder.Uri.ToString();
}
private static string FilterTrackingParameters(string query, Uri uri, string language)
{
if (string.IsNullOrWhiteSpace(query))
{
return string.Empty;
}
var trimmed = query.TrimStart('?');
if (string.IsNullOrWhiteSpace(trimmed))
{
return string.Empty;
}
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
var kept = new List<string>();
foreach (var parameter in parameters)
{
var separatorIndex = parameter.IndexOf('=');
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
{
kept.Add($"lang={language}");
continue;
}
kept.Add(parameter);
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
{
kept.Add($"lang={language}");
}
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
}
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
{
if (document.Body is null)
{
return Array.Empty<string>();
}
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
if (matches.Count == 0)
{
return Array.Empty<string>();
}
return matches
.Select(match => match.Value.ToUpperInvariant())
.Distinct(StringComparer.Ordinal)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
}
private static string? CollapseWhitespace(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
return collapsed.Length == 0 ? null : collapsed;
}
}

View File

@@ -1,258 +1,258 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Normalization.SemVer;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal static class CccsMapper
{
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"advisory",
dto.AlertType ?? dto.SerialNumber,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory })
};
return new Advisory(
advisoryKey: dto.SerialNumber,
title: dto.Title,
summary: dto.Summary,
language: dto.Language,
published: dto.Published ?? dto.Modified,
modified: dto.Modified ?? dto.Published,
severity: null,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
}
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4)
{
dto.SerialNumber,
};
if (!string.IsNullOrWhiteSpace(dto.SourceId)
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
{
aliases.Add(dto.SourceId);
}
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
dto.CanonicalUrl,
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
"reference",
"cccs",
reference.Label,
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
for (var index = 0; index < dto.Products.Count; index++)
{
var product = dto.Products[index];
if (string.IsNullOrWhiteSpace(product))
{
continue;
}
var identifier = product.Trim();
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var rangeAnchor = $"cccs:{dto.SerialNumber}:{index}";
var versionRanges = BuildVersionRanges(product, rangeAnchor, recordedAt);
var normalizedVersions = BuildNormalizedVersions(versionRanges, rangeAnchor);
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: versionRanges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedVersions));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(string productText, string rangeAnchor, DateTimeOffset recordedAt)
{
var versionText = ExtractFirstVersionToken(productText);
if (string.IsNullOrWhiteSpace(versionText))
{
return Array.Empty<AffectedVersionRange>();
}
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"range",
rangeAnchor,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges });
var vendorExtensions = new Dictionary<string, string>
{
["cccs.version.raw"] = versionText!,
["cccs.anchor"] = rangeAnchor,
};
var semVerResults = SemVerRangeRuleBuilder.Build(versionText!, patchedVersion: null, provenanceNote: rangeAnchor);
if (semVerResults.Count > 0)
{
return semVerResults.Select(result =>
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: result.Primitive.Introduced,
fixedVersion: result.Primitive.Fixed,
lastAffectedVersion: result.Primitive.LastAffected,
rangeExpression: result.Expression ?? versionText!,
provenance: provenance,
primitives: new RangePrimitives(
result.Primitive,
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions)))
.ToArray();
}
var primitives = new RangePrimitives(
new SemVerPrimitive(
Introduced: versionText,
IntroducedInclusive: true,
Fixed: null,
FixedInclusive: false,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null,
ExactValue: versionText),
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions);
return new[]
{
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: versionText,
provenance: provenance,
primitives: primitives),
};
}
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
IReadOnlyList<AffectedVersionRange> ranges,
string rangeAnchor)
{
if (ranges.Count == 0)
{
return Array.Empty<NormalizedVersionRule>();
}
var rules = new List<NormalizedVersionRule>(ranges.Count);
foreach (var range in ranges)
{
var rule = range.ToNormalizedVersionRule(rangeAnchor);
if (rule is not null)
{
rules.Add(rule);
}
}
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
}
private static string? ExtractFirstVersionToken(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var match = Regex.Match(value, @"\d+(?:\.\d+){0,3}(?:[A-Za-z0-9\-_]*)?");
return match.Success ? match.Value : null;
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Normalization.SemVer;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal static class CccsMapper
{
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"advisory",
dto.AlertType ?? dto.SerialNumber,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory })
};
return new Advisory(
advisoryKey: dto.SerialNumber,
title: dto.Title,
summary: dto.Summary,
language: dto.Language,
published: dto.Published ?? dto.Modified,
modified: dto.Modified ?? dto.Published,
severity: null,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
}
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4)
{
dto.SerialNumber,
};
if (!string.IsNullOrWhiteSpace(dto.SourceId)
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
{
aliases.Add(dto.SourceId);
}
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
dto.CanonicalUrl,
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
"reference",
"cccs",
reference.Label,
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
for (var index = 0; index < dto.Products.Count; index++)
{
var product = dto.Products[index];
if (string.IsNullOrWhiteSpace(product))
{
continue;
}
var identifier = product.Trim();
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var rangeAnchor = $"cccs:{dto.SerialNumber}:{index}";
var versionRanges = BuildVersionRanges(product, rangeAnchor, recordedAt);
var normalizedVersions = BuildNormalizedVersions(versionRanges, rangeAnchor);
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: versionRanges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedVersions));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(string productText, string rangeAnchor, DateTimeOffset recordedAt)
{
var versionText = ExtractFirstVersionToken(productText);
if (string.IsNullOrWhiteSpace(versionText))
{
return Array.Empty<AffectedVersionRange>();
}
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"range",
rangeAnchor,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges });
var vendorExtensions = new Dictionary<string, string>
{
["cccs.version.raw"] = versionText!,
["cccs.anchor"] = rangeAnchor,
};
var semVerResults = SemVerRangeRuleBuilder.Build(versionText!, patchedVersion: null, provenanceNote: rangeAnchor);
if (semVerResults.Count > 0)
{
return semVerResults.Select(result =>
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: result.Primitive.Introduced,
fixedVersion: result.Primitive.Fixed,
lastAffectedVersion: result.Primitive.LastAffected,
rangeExpression: result.Expression ?? versionText!,
provenance: provenance,
primitives: new RangePrimitives(
result.Primitive,
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions)))
.ToArray();
}
var primitives = new RangePrimitives(
new SemVerPrimitive(
Introduced: versionText,
IntroducedInclusive: true,
Fixed: null,
FixedInclusive: false,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null,
ExactValue: versionText),
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions);
return new[]
{
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: versionText,
provenance: provenance,
primitives: primitives),
};
}
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
IReadOnlyList<AffectedVersionRange> ranges,
string rangeAnchor)
{
if (ranges.Count == 0)
{
return Array.Empty<NormalizedVersionRule>();
}
var rules = new List<NormalizedVersionRule>(ranges.Count);
foreach (var range in ranges)
{
var rule = range.ToNormalizedVersionRule(rangeAnchor);
if (rule is not null)
{
rules.Add(rule);
}
}
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
}
private static string? ExtractFirstVersionToken(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var match = Regex.Match(value, @"\d+(?:\.\d+){0,3}(?:[A-Za-z0-9\-_]*)?");
return match.Success ? match.Value : null;
}
}

View File

@@ -1,58 +1,58 @@
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsRawAdvisoryDocument
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string? SerialNumber { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("externalUrl")]
public string? ExternalUrl { get; init; }
[JsonPropertyName("bodyHtml")]
public string BodyHtml { get; init; } = string.Empty;
[JsonPropertyName("bodySegments")]
public string[] BodySegments { get; init; } = Array.Empty<string>();
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("rawCreated")]
public string? RawDateCreated { get; init; }
[JsonPropertyName("rawModified")]
public string? RawDateModified { get; init; }
}
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsRawAdvisoryDocument
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string? SerialNumber { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("externalUrl")]
public string? ExternalUrl { get; init; }
[JsonPropertyName("bodyHtml")]
public string BodyHtml { get; init; } = string.Empty;
[JsonPropertyName("bodySegments")]
public string[] BodySegments { get; init; } = Array.Empty<string>();
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("rawCreated")]
public string? RawDateCreated { get; init; }
[JsonPropertyName("rawModified")]
public string? RawDateModified { get; init; }
}

View File

@@ -1,22 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Cccs;
internal static class CccsJobKinds
{
public const string Fetch = "source:cccs:fetch";
}
internal sealed class CccsFetchJob : IJob
{
private readonly CccsConnector _connector;
public CccsFetchJob(CccsConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Cccs;
internal static class CccsJobKinds
{
public const string Fetch = "source:cccs:fetch";
}
internal sealed class CccsFetchJob : IJob
{
private readonly CccsConnector _connector;
public CccsFetchJob(CccsConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")]

View File

@@ -6,7 +6,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.CertBund.Configuration;
using StellaOps.Concelier.Connector.CertBund.Internal;
using StellaOps.Concelier.Connector.Common;
@@ -286,7 +286,7 @@ public sealed class CertBundConnector : IFeedConnector
_diagnostics.ParseSuccess(dto.Products.Count, dto.CveIds.Count);
parsedCount++;
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var bson = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", bson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -428,7 +428,7 @@ public sealed class CertBundConnector : IFeedConnector
private Task UpdateCursorAsync(CertBundCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

View File

@@ -1,6 +1,6 @@
using System;
using System.Linq;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
@@ -31,13 +31,13 @@ internal sealed record CertBundCursor(
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
public DocumentObject ToDocumentObject()
{
var document = new BsonDocument
var document = new DocumentObject
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new BsonArray(KnownAdvisories),
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new DocumentArray(KnownAdvisories),
};
if (LastPublished.HasValue)
@@ -53,7 +53,7 @@ internal sealed record CertBundCursor(
return document;
}
public static CertBundCursor FromBson(BsonDocument? document)
public static CertBundCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
@@ -76,9 +76,9 @@ internal sealed record CertBundCursor(
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuids;
}
@@ -95,9 +95,9 @@ internal sealed record CertBundCursor(
return items;
}
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyStrings;
}
@@ -108,11 +108,11 @@ internal sealed record CertBundCursor(
.ToArray();
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -9,7 +9,7 @@ using System.Text.Json.Serialization;
using System.Threading;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
@@ -338,7 +338,7 @@ public sealed class CertCcConnector : IFeedConnector
var dto = CertCcNoteParser.Parse(noteBytes, vendorsBytes, vulsBytes, vendorStatusesBytes);
var json = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var payload = StellaOps.Concelier.Bson.BsonDocument.Parse(json);
var payload = StellaOps.Concelier.Documents.DocumentObject.Parse(json);
_diagnostics.ParseSuccess(
dto.Vendors.Count,
@@ -678,7 +678,7 @@ public sealed class CertCcConnector : IFeedConnector
private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
private sealed class NoteDocumentGroup

View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-cc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertCcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertCcConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-cc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertCcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertCcConnector>();
}
}

View File

@@ -1,50 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertCc.Configuration;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-cc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertCcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertCcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertCc.Configuration;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-cc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertCcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertCcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -1,37 +1,37 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertCc;
public static class CertCcServiceCollectionExtensions
{
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertCcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
clientOptions.BaseAddress = options.BaseApiUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
});
services.TryAddSingleton<CertCcSummaryPlanner>();
services.TryAddSingleton<CertCcDiagnostics>();
services.AddTransient<CertCcConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertCc;
public static class CertCcServiceCollectionExtensions
{
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertCcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
clientOptions.BaseAddress = options.BaseApiUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
});
services.TryAddSingleton<CertCcSummaryPlanner>();
services.TryAddSingleton<CertCcDiagnostics>();
services.AddTransient<CertCcConnector>();
return services;
}
}

View File

@@ -1,79 +1,79 @@
using System;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
/// <summary>
/// Connector options governing CERT/CC fetch cadence and API endpoints.
/// </summary>
public sealed class CertCcOptions
{
public const string HttpClientName = "certcc";
/// <summary>
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
/// </summary>
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
/// <summary>
/// Sliding window settings controlling which summary endpoints are requested.
/// </summary>
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
{
WindowSize = TimeSpan.FromDays(30),
Overlap = TimeSpan.FromDays(3),
InitialBackfill = TimeSpan.FromDays(365),
MinimumWindowSize = TimeSpan.FromDays(1),
};
/// <summary>
/// Maximum number of monthly summary endpoints to request in a single plan.
/// </summary>
public int MaxMonthlySummaries { get; set; } = 6;
/// <summary>
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
/// </summary>
public int MaxNotesPerFetch { get; set; } = 25;
/// <summary>
/// Optional delay inserted between successive detail requests to respect upstream throttling.
/// </summary>
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;
public void Validate()
{
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
}
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
}
SummaryWindow ??= new TimeWindowCursorOptions();
SummaryWindow.EnsureValid();
if (MaxMonthlySummaries <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
}
if (MaxNotesPerFetch <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
}
if (DetailRequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
}
}
}
using System;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
/// <summary>
/// Connector options governing CERT/CC fetch cadence and API endpoints.
/// </summary>
public sealed class CertCcOptions
{
public const string HttpClientName = "certcc";
/// <summary>
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
/// </summary>
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
/// <summary>
/// Sliding window settings controlling which summary endpoints are requested.
/// </summary>
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
{
WindowSize = TimeSpan.FromDays(30),
Overlap = TimeSpan.FromDays(3),
InitialBackfill = TimeSpan.FromDays(365),
MinimumWindowSize = TimeSpan.FromDays(1),
};
/// <summary>
/// Maximum number of monthly summary endpoints to request in a single plan.
/// </summary>
public int MaxMonthlySummaries { get; set; } = 6;
/// <summary>
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
/// </summary>
public int MaxNotesPerFetch { get; set; } = 25;
/// <summary>
/// Optional delay inserted between successive detail requests to respect upstream throttling.
/// </summary>
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;
public void Validate()
{
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
}
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
}
SummaryWindow ??= new TimeWindowCursorOptions();
SummaryWindow.EnsureValid();
if (MaxMonthlySummaries <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
}
if (MaxNotesPerFetch <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
}
if (DetailRequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
}
}
}

View File

@@ -1,4 +1,4 @@
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
@@ -22,18 +22,18 @@ internal sealed record CertCcCursor(
EmptyGuidArray,
null);
public BsonDocument ToBsonDocument()
public DocumentObject ToDocumentObject()
{
var document = new BsonDocument();
var document = new DocumentObject();
var summary = new BsonDocument();
var summary = new DocumentObject();
SummaryState.WriteTo(summary, "start", "end");
document["summary"] = summary;
document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
document["pendingSummaries"] = new DocumentArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new DocumentArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString()));
if (LastRun.HasValue)
{
@@ -43,7 +43,7 @@ internal sealed record CertCcCursor(
return document;
}
public static CertCcCursor FromBson(BsonDocument? document)
public static CertCcCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
@@ -51,9 +51,9 @@ internal sealed record CertCcCursor(
}
TimeWindowCursorState summaryState = TimeWindowCursorState.Empty;
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument)
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is DocumentObject summaryDocument)
{
summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end");
summaryState = TimeWindowCursorState.FromDocumentObject(summaryDocument, "start", "end");
}
var pendingSummaries = ReadGuidArray(document, "pendingSummaries");
@@ -64,10 +64,10 @@ internal sealed record CertCcCursor(
DateTimeOffset? lastRun = null;
if (document.TryGetValue("lastRun", out var lastRunValue))
{
lastRun = lastRunValue.BsonType switch
lastRun = lastRunValue.DocumentType switch
{
BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
@@ -93,9 +93,9 @@ internal sealed record CertCcCursor(
public CertCcCursor WithLastRun(DateTimeOffset? timestamp)
=> this with { LastRun = timestamp };
private static Guid[] ReadGuidArray(BsonDocument document, string field)
private static Guid[] ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyGuidArray;
}
@@ -112,9 +112,9 @@ internal sealed record CertCcCursor(
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
}
private static string[] ReadStringArray(BsonDocument document, string field)
private static string[] ReadStringArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyStringArray;
}
@@ -124,10 +124,10 @@ internal sealed record CertCcCursor(
{
switch (element)
{
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
case DocumentString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
results.Add(bsonString.AsString.Trim());
break;
case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
case DocumentObject bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
results.Add(inner.AsString.Trim());
break;
}
@@ -142,14 +142,14 @@ internal sealed record CertCcCursor(
.ToArray();
}
private static bool TryReadGuid(BsonValue value, out Guid guid)
private static bool TryReadGuid(DocumentValue value, out Guid guid)
{
if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
if (value is DocumentString bsonString && Guid.TryParse(bsonString.AsString, out guid))
{
return true;
}
if (value is BsonBinaryData binary)
if (value is DocumentBinaryData binary)
{
try
{

View File

@@ -1,214 +1,214 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
/// </summary>
public sealed class CertCcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _planWindows;
private readonly Counter<long> _planRequests;
private readonly Histogram<double> _planWindowDays;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchUnchanged;
private readonly Counter<long> _summaryFetchFailures;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchUnchanged;
private readonly Counter<long> _detailFetchMissing;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseVendorCount;
private readonly Histogram<long> _parseStatusCount;
private readonly Histogram<long> _parseVulnerabilityCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffectedPackageCount;
private readonly Histogram<long> _mapNormalizedVersionCount;
public CertCcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_planWindows = _meter.CreateCounter<long>(
name: "certcc.plan.windows",
unit: "windows",
description: "Number of summary planning windows evaluated.");
_planRequests = _meter.CreateCounter<long>(
name: "certcc.plan.requests",
unit: "requests",
description: "Total CERT/CC summary endpoints queued by the planner.");
_planWindowDays = _meter.CreateHistogram<double>(
name: "certcc.plan.window_days",
unit: "day",
description: "Duration of each planning window in days.");
_summaryFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.attempts",
unit: "operations",
description: "Number of VINCE summary fetch attempts.");
_summaryFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.success",
unit: "operations",
description: "Number of VINCE summary fetches persisted to storage.");
_summaryFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.not_modified",
unit: "operations",
description: "Number of VINCE summary fetches returning HTTP 304.");
_summaryFetchFailures = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.failures",
unit: "operations",
description: "Number of VINCE summary fetches that failed after retries.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.attempts",
unit: "operations",
description: "Number of VINCE detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.success",
unit: "operations",
description: "Number of VINCE detail fetches that returned payloads.");
_detailFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.unchanged",
unit: "operations",
description: "Number of VINCE detail fetches returning HTTP 304.");
_detailFetchMissing = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.missing",
unit: "operations",
description: "Number of optional VINCE detail endpoints missing but tolerated.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.failures",
unit: "operations",
description: "Number of VINCE detail fetches that failed after retries.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certcc.parse.success",
unit: "documents",
description: "Number of VINCE note bundles parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certcc.parse.failures",
unit: "documents",
description: "Number of VINCE note bundles that failed to parse.");
_parseVendorCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vendors.count",
unit: "vendors",
description: "Distribution of vendor statements per VINCE note.");
_parseStatusCount = _meter.CreateHistogram<long>(
name: "certcc.parse.statuses.count",
unit: "entries",
description: "Distribution of vendor status entries per VINCE note.");
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vulnerabilities.count",
unit: "entries",
description: "Distribution of vulnerability records per VINCE note.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certcc.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certcc.map.failures",
unit: "advisories",
description: "Number of CERT/CC advisory mapping attempts that failed.");
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
name: "certcc.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per CERT/CC advisory.");
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
name: "certcc.map.normalized_versions.count",
unit: "rules",
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
}
public void PlanEvaluated(TimeWindow window, int requestCount)
{
_planWindows.Add(1);
if (requestCount > 0)
{
_planRequests.Add(requestCount);
}
var duration = window.Duration;
if (duration > TimeSpan.Zero)
{
_planWindowDays.Record(duration.TotalDays);
}
}
public void SummaryFetchAttempt(CertCcSummaryScope scope)
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
public void SummaryFetchSuccess(CertCcSummaryScope scope)
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
public void SummaryFetchFailure(CertCcSummaryScope scope)
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
public void DetailFetchAttempt(string endpoint)
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
public void DetailFetchSuccess(string endpoint)
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
public void DetailFetchUnchanged(string endpoint)
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
public void DetailFetchMissing(string endpoint)
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
public void DetailFetchFailure(string endpoint)
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
{
_parseSuccess.Add(1);
if (vendorCount >= 0)
{
_parseVendorCount.Record(vendorCount);
}
if (statusCount >= 0)
{
_parseStatusCount.Record(statusCount);
}
if (vulnerabilityCount >= 0)
{
_parseVulnerabilityCount.Record(vulnerabilityCount);
}
}
public void ParseFailure()
=> _parseFailures.Add(1);
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
{
_mapSuccess.Add(1);
if (affectedPackageCount >= 0)
{
_mapAffectedPackageCount.Record(affectedPackageCount);
}
if (normalizedVersionCount >= 0)
{
_mapNormalizedVersionCount.Record(normalizedVersionCount);
}
}
public void MapFailure()
=> _mapFailures.Add(1);
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
=> new("scope", scope.ToString().ToLowerInvariant());
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
/// </summary>
public sealed class CertCcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _planWindows;
private readonly Counter<long> _planRequests;
private readonly Histogram<double> _planWindowDays;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchUnchanged;
private readonly Counter<long> _summaryFetchFailures;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchUnchanged;
private readonly Counter<long> _detailFetchMissing;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseVendorCount;
private readonly Histogram<long> _parseStatusCount;
private readonly Histogram<long> _parseVulnerabilityCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffectedPackageCount;
private readonly Histogram<long> _mapNormalizedVersionCount;
public CertCcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_planWindows = _meter.CreateCounter<long>(
name: "certcc.plan.windows",
unit: "windows",
description: "Number of summary planning windows evaluated.");
_planRequests = _meter.CreateCounter<long>(
name: "certcc.plan.requests",
unit: "requests",
description: "Total CERT/CC summary endpoints queued by the planner.");
_planWindowDays = _meter.CreateHistogram<double>(
name: "certcc.plan.window_days",
unit: "day",
description: "Duration of each planning window in days.");
_summaryFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.attempts",
unit: "operations",
description: "Number of VINCE summary fetch attempts.");
_summaryFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.success",
unit: "operations",
description: "Number of VINCE summary fetches persisted to storage.");
_summaryFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.not_modified",
unit: "operations",
description: "Number of VINCE summary fetches returning HTTP 304.");
_summaryFetchFailures = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.failures",
unit: "operations",
description: "Number of VINCE summary fetches that failed after retries.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.attempts",
unit: "operations",
description: "Number of VINCE detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.success",
unit: "operations",
description: "Number of VINCE detail fetches that returned payloads.");
_detailFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.unchanged",
unit: "operations",
description: "Number of VINCE detail fetches returning HTTP 304.");
_detailFetchMissing = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.missing",
unit: "operations",
description: "Number of optional VINCE detail endpoints missing but tolerated.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.failures",
unit: "operations",
description: "Number of VINCE detail fetches that failed after retries.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certcc.parse.success",
unit: "documents",
description: "Number of VINCE note bundles parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certcc.parse.failures",
unit: "documents",
description: "Number of VINCE note bundles that failed to parse.");
_parseVendorCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vendors.count",
unit: "vendors",
description: "Distribution of vendor statements per VINCE note.");
_parseStatusCount = _meter.CreateHistogram<long>(
name: "certcc.parse.statuses.count",
unit: "entries",
description: "Distribution of vendor status entries per VINCE note.");
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vulnerabilities.count",
unit: "entries",
description: "Distribution of vulnerability records per VINCE note.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certcc.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certcc.map.failures",
unit: "advisories",
description: "Number of CERT/CC advisory mapping attempts that failed.");
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
name: "certcc.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per CERT/CC advisory.");
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
name: "certcc.map.normalized_versions.count",
unit: "rules",
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
}
public void PlanEvaluated(TimeWindow window, int requestCount)
{
_planWindows.Add(1);
if (requestCount > 0)
{
_planRequests.Add(requestCount);
}
var duration = window.Duration;
if (duration > TimeSpan.Zero)
{
_planWindowDays.Record(duration.TotalDays);
}
}
public void SummaryFetchAttempt(CertCcSummaryScope scope)
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
public void SummaryFetchSuccess(CertCcSummaryScope scope)
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
public void SummaryFetchFailure(CertCcSummaryScope scope)
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
public void DetailFetchAttempt(string endpoint)
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
public void DetailFetchSuccess(string endpoint)
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
public void DetailFetchUnchanged(string endpoint)
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
public void DetailFetchMissing(string endpoint)
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
public void DetailFetchFailure(string endpoint)
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
{
_parseSuccess.Add(1);
if (vendorCount >= 0)
{
_parseVendorCount.Record(vendorCount);
}
if (statusCount >= 0)
{
_parseStatusCount.Record(statusCount);
}
if (vulnerabilityCount >= 0)
{
_parseVulnerabilityCount.Record(vulnerabilityCount);
}
}
public void ParseFailure()
=> _parseFailures.Add(1);
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
{
_mapSuccess.Add(1);
if (affectedPackageCount >= 0)
{
_mapAffectedPackageCount.Record(affectedPackageCount);
}
if (normalizedVersionCount >= 0)
{
_mapNormalizedVersionCount.Record(normalizedVersionCount);
}
}
public void MapFailure()
=> _mapFailures.Add(1);
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
=> new("scope", scope.ToString().ToLowerInvariant());
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -1,97 +1,97 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcNoteDto(
CertCcNoteMetadata Metadata,
IReadOnlyList<CertCcVendorDto> Vendors,
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
{
public static CertCcNoteDto Empty { get; } = new(
CertCcNoteMetadata.Empty,
Array.Empty<CertCcVendorDto>(),
Array.Empty<CertCcVendorStatusDto>(),
Array.Empty<CertCcVulnerabilityDto>());
}
internal sealed record CertCcNoteMetadata(
string? VuId,
string IdNumber,
string Title,
string? Overview,
string? Summary,
DateTimeOffset? Published,
DateTimeOffset? Updated,
DateTimeOffset? Created,
int? Revision,
IReadOnlyList<string> CveIds,
IReadOnlyList<string> PublicUrls,
string? PrimaryUrl)
{
public static CertCcNoteMetadata Empty { get; } = new(
VuId: null,
IdNumber: string.Empty,
Title: string.Empty,
Overview: null,
Summary: null,
Published: null,
Updated: null,
Created: null,
Revision: null,
CveIds: Array.Empty<string>(),
PublicUrls: Array.Empty<string>(),
PrimaryUrl: null);
}
internal sealed record CertCcVendorDto(
string Vendor,
DateTimeOffset? ContactDate,
DateTimeOffset? StatementDate,
DateTimeOffset? Updated,
string? Statement,
string? Addendum,
IReadOnlyList<string> References)
{
public static CertCcVendorDto Empty { get; } = new(
Vendor: string.Empty,
ContactDate: null,
StatementDate: null,
Updated: null,
Statement: null,
Addendum: null,
References: Array.Empty<string>());
}
internal sealed record CertCcVendorStatusDto(
string Vendor,
string CveId,
string Status,
string? Statement,
IReadOnlyList<string> References,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVendorStatusDto Empty { get; } = new(
Vendor: string.Empty,
CveId: string.Empty,
Status: string.Empty,
Statement: null,
References: Array.Empty<string>(),
DateAdded: null,
DateUpdated: null);
}
internal sealed record CertCcVulnerabilityDto(
string CveId,
string? Description,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVulnerabilityDto Empty { get; } = new(
CveId: string.Empty,
Description: null,
DateAdded: null,
DateUpdated: null);
}
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcNoteDto(
CertCcNoteMetadata Metadata,
IReadOnlyList<CertCcVendorDto> Vendors,
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
{
public static CertCcNoteDto Empty { get; } = new(
CertCcNoteMetadata.Empty,
Array.Empty<CertCcVendorDto>(),
Array.Empty<CertCcVendorStatusDto>(),
Array.Empty<CertCcVulnerabilityDto>());
}
internal sealed record CertCcNoteMetadata(
string? VuId,
string IdNumber,
string Title,
string? Overview,
string? Summary,
DateTimeOffset? Published,
DateTimeOffset? Updated,
DateTimeOffset? Created,
int? Revision,
IReadOnlyList<string> CveIds,
IReadOnlyList<string> PublicUrls,
string? PrimaryUrl)
{
public static CertCcNoteMetadata Empty { get; } = new(
VuId: null,
IdNumber: string.Empty,
Title: string.Empty,
Overview: null,
Summary: null,
Published: null,
Updated: null,
Created: null,
Revision: null,
CveIds: Array.Empty<string>(),
PublicUrls: Array.Empty<string>(),
PrimaryUrl: null);
}
internal sealed record CertCcVendorDto(
string Vendor,
DateTimeOffset? ContactDate,
DateTimeOffset? StatementDate,
DateTimeOffset? Updated,
string? Statement,
string? Addendum,
IReadOnlyList<string> References)
{
public static CertCcVendorDto Empty { get; } = new(
Vendor: string.Empty,
ContactDate: null,
StatementDate: null,
Updated: null,
Statement: null,
Addendum: null,
References: Array.Empty<string>());
}
internal sealed record CertCcVendorStatusDto(
string Vendor,
string CveId,
string Status,
string? Statement,
IReadOnlyList<string> References,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVendorStatusDto Empty { get; } = new(
Vendor: string.Empty,
CveId: string.Empty,
Status: string.Empty,
Statement: null,
References: Array.Empty<string>(),
DateAdded: null,
DateUpdated: null);
}
internal sealed record CertCcVulnerabilityDto(
string CveId,
string? Description,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVulnerabilityDto Empty { get; } = new(
CveId: string.Empty,
Description: null,
DateAdded: null,
DateUpdated: null);
}

View File

@@ -1,108 +1,108 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcSummaryParser
{
public static IReadOnlyList<string> ParseNotes(byte[] payload)
{
if (payload is null || payload.Length == 0)
{
return Array.Empty<string>();
}
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
});
var notesElement = document.RootElement.ValueKind switch
{
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
JsonValueKind.Array => document.RootElement,
JsonValueKind.Null or JsonValueKind.Undefined => default,
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
};
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
{
return Array.Empty<string>();
}
var results = new List<string>(notesElement.GetArrayLength());
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var element in notesElement.EnumerateArray())
{
var token = ExtractToken(element);
if (string.IsNullOrWhiteSpace(token))
{
continue;
}
var normalized = token.Trim();
var dedupKey = CreateDedupKey(normalized);
if (seen.Add(dedupKey))
{
results.Add(normalized);
}
}
return results.Count == 0 ? Array.Empty<string>() : results;
}
private static string CreateDedupKey(string token)
{
var digits = string.Concat(token.Where(char.IsDigit));
return digits.Length > 0
? digits
: token.Trim().ToUpperInvariant();
}
private static string? ExtractToken(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number => element.TryGetInt64(out var number)
? number.ToString(CultureInfo.InvariantCulture)
: element.GetRawText(),
JsonValueKind.Object => ExtractFromObject(element),
_ => null,
};
}
private static string? ExtractFromObject(JsonElement element)
{
foreach (var propertyName in PropertyCandidates)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
{
var value = property.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
return null;
}
private static readonly string[] PropertyCandidates =
{
"note",
"notes",
"id",
"idnumber",
"noteId",
"vu",
"vuid",
"vuId",
};
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcSummaryParser
{
public static IReadOnlyList<string> ParseNotes(byte[] payload)
{
if (payload is null || payload.Length == 0)
{
return Array.Empty<string>();
}
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
});
var notesElement = document.RootElement.ValueKind switch
{
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
JsonValueKind.Array => document.RootElement,
JsonValueKind.Null or JsonValueKind.Undefined => default,
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
};
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
{
return Array.Empty<string>();
}
var results = new List<string>(notesElement.GetArrayLength());
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var element in notesElement.EnumerateArray())
{
var token = ExtractToken(element);
if (string.IsNullOrWhiteSpace(token))
{
continue;
}
var normalized = token.Trim();
var dedupKey = CreateDedupKey(normalized);
if (seen.Add(dedupKey))
{
results.Add(normalized);
}
}
return results.Count == 0 ? Array.Empty<string>() : results;
}
private static string CreateDedupKey(string token)
{
var digits = string.Concat(token.Where(char.IsDigit));
return digits.Length > 0
? digits
: token.Trim().ToUpperInvariant();
}
private static string? ExtractToken(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number => element.TryGetInt64(out var number)
? number.ToString(CultureInfo.InvariantCulture)
: element.GetRawText(),
JsonValueKind.Object => ExtractFromObject(element),
_ => null,
};
}
private static string? ExtractFromObject(JsonElement element)
{
foreach (var propertyName in PropertyCandidates)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
{
var value = property.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
return null;
}
private static readonly string[] PropertyCandidates =
{
"note",
"notes",
"id",
"idnumber",
"noteId",
"vu",
"vuid",
"vuId",
};
}

View File

@@ -1,22 +1,22 @@
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
public sealed record CertCcSummaryPlan(
TimeWindow Window,
IReadOnlyList<CertCcSummaryRequest> Requests,
TimeWindowCursorState NextState);
public enum CertCcSummaryScope
{
Monthly,
Yearly,
}
public sealed record CertCcSummaryRequest(
Uri Uri,
CertCcSummaryScope Scope,
int Year,
int? Month);
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
public sealed record CertCcSummaryPlan(
TimeWindow Window,
IReadOnlyList<CertCcSummaryRequest> Requests,
TimeWindowCursorState NextState);
public enum CertCcSummaryScope
{
Monthly,
Yearly,
}
public sealed record CertCcSummaryRequest(
Uri Uri,
CertCcSummaryScope Scope,
int Year,
int? Month);

View File

@@ -1,96 +1,96 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
/// </summary>
public sealed class CertCcSummaryPlanner
{
private readonly CertCcOptions _options;
private readonly TimeProvider _timeProvider;
public CertCcSummaryPlanner(
IOptions<CertCcOptions> options,
TimeProvider? timeProvider = null)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
}
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
{
var now = _timeProvider.GetUtcNow();
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
var months = EnumerateYearMonths(window.Start, window.End)
.Take(_options.MaxMonthlySummaries)
.ToArray();
if (months.Length == 0)
{
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
}
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
foreach (var month in months)
{
requests.Add(new CertCcSummaryRequest(
BuildMonthlyUri(month.Year, month.Month),
CertCcSummaryScope.Monthly,
month.Year,
month.Month));
}
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
{
requests.Add(new CertCcSummaryRequest(
BuildYearlyUri(year),
CertCcSummaryScope.Yearly,
year,
Month: null));
}
return new CertCcSummaryPlan(window, requests, nextState);
}
private Uri BuildMonthlyUri(int year, int month)
{
var path = $"{year:D4}/{month:D2}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private Uri BuildYearlyUri(int year)
{
var path = $"{year:D4}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
{
if (end <= start)
{
yield break;
}
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
{
limit = limit.AddMonths(1);
}
while (cursor < limit)
{
yield return (cursor.Year, cursor.Month);
cursor = cursor.AddMonths(1);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
/// </summary>
public sealed class CertCcSummaryPlanner
{
private readonly CertCcOptions _options;
private readonly TimeProvider _timeProvider;
public CertCcSummaryPlanner(
IOptions<CertCcOptions> options,
TimeProvider? timeProvider = null)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
}
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
{
var now = _timeProvider.GetUtcNow();
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
var months = EnumerateYearMonths(window.Start, window.End)
.Take(_options.MaxMonthlySummaries)
.ToArray();
if (months.Length == 0)
{
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
}
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
foreach (var month in months)
{
requests.Add(new CertCcSummaryRequest(
BuildMonthlyUri(month.Year, month.Month),
CertCcSummaryScope.Monthly,
month.Year,
month.Month));
}
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
{
requests.Add(new CertCcSummaryRequest(
BuildYearlyUri(year),
CertCcSummaryScope.Yearly,
year,
Month: null));
}
return new CertCcSummaryPlan(window, requests, nextState);
}
private Uri BuildMonthlyUri(int year, int month)
{
var path = $"{year:D4}/{month:D2}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private Uri BuildYearlyUri(int year)
{
var path = $"{year:D4}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
{
if (end <= start)
{
yield break;
}
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
{
limit = limit.AddMonths(1);
}
while (cursor < limit)
{
yield return (cursor.Year, cursor.Month);
cursor = cursor.AddMonths(1);
}
}
}

View File

@@ -1,235 +1,235 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcVendorStatementParser
{
private static readonly string[] PairSeparators =
{
"\t",
" - ",
" ",
" — ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
{
if (string.IsNullOrWhiteSpace(statement))
{
return Array.Empty<CertCcVendorPatch>();
}
var patches = new List<CertCcVendorPatch>();
var lines = statement
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
if (line.Length == 0)
{
continue;
}
line = TrimBulletPrefix(line);
if (line.Length == 0)
{
continue;
}
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
{
continue;
}
var versions = ExtractVersions(versionSegment);
if (versions.Count == 0)
{
continue;
}
var products = ExtractProducts(productSegment);
if (products.Count == 0)
{
products.Add(string.Empty);
}
if (versions.Count == products.Count)
{
for (var index = 0; index < versions.Count; index++)
{
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
}
continue;
}
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
{
var groupSize = products.Count / versions.Count;
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
{
var start = versionIndex * groupSize;
var end = start + groupSize;
var version = versions[versionIndex];
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
{
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
}
}
continue;
}
var primaryVersion = versions[0];
foreach (var product in products)
{
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
}
}
if (patches.Count == 0)
{
return Array.Empty<CertCcVendorPatch>();
}
return patches
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
.Distinct(CertCcVendorPatch.Comparer)
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
.ToArray();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart(BulletPrefixes).Trim();
return trimmed.Length == 0 ? value.Trim() : trimmed;
}
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
{
foreach (var separator in PairSeparators)
{
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
productSegment = parts[0];
versionSegment = parts[1];
return true;
}
}
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (whitespaceSplit.Length >= 2)
{
productSegment = string.Join(' ', whitespaceSplit[..^1]);
versionSegment = whitespaceSplit[^1];
return true;
}
productSegment = string.Empty;
versionSegment = string.Empty;
return false;
}
private static List<string> ExtractProducts(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var normalized = segment.Replace('\t', ' ').Trim();
var tokens = normalized
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
.Select(static token => token.Trim())
.Where(static token => token.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return tokens;
}
private static List<string> ExtractVersions(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var matches = VersionTokenRegex.Matches(segment);
if (matches.Count == 0)
{
return new List<string>();
}
var versions = new List<string>(matches.Count);
foreach (Match match in matches)
{
if (match.Groups.Count == 0)
{
continue;
}
var value = match.Groups[1].Value.Trim();
if (value.Length == 0)
{
continue;
}
versions.Add(value);
}
return versions
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(32)
.ToList();
}
}
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
{
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
{
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
}
public int GetHashCode(CertCcVendorPatch obj)
{
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
return HashCode.Combine(product, version);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcVendorStatementParser
{
private static readonly string[] PairSeparators =
{
"\t",
" - ",
" ",
" — ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
{
if (string.IsNullOrWhiteSpace(statement))
{
return Array.Empty<CertCcVendorPatch>();
}
var patches = new List<CertCcVendorPatch>();
var lines = statement
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
if (line.Length == 0)
{
continue;
}
line = TrimBulletPrefix(line);
if (line.Length == 0)
{
continue;
}
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
{
continue;
}
var versions = ExtractVersions(versionSegment);
if (versions.Count == 0)
{
continue;
}
var products = ExtractProducts(productSegment);
if (products.Count == 0)
{
products.Add(string.Empty);
}
if (versions.Count == products.Count)
{
for (var index = 0; index < versions.Count; index++)
{
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
}
continue;
}
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
{
var groupSize = products.Count / versions.Count;
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
{
var start = versionIndex * groupSize;
var end = start + groupSize;
var version = versions[versionIndex];
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
{
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
}
}
continue;
}
var primaryVersion = versions[0];
foreach (var product in products)
{
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
}
}
if (patches.Count == 0)
{
return Array.Empty<CertCcVendorPatch>();
}
return patches
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
.Distinct(CertCcVendorPatch.Comparer)
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
.ToArray();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart(BulletPrefixes).Trim();
return trimmed.Length == 0 ? value.Trim() : trimmed;
}
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
{
foreach (var separator in PairSeparators)
{
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
productSegment = parts[0];
versionSegment = parts[1];
return true;
}
}
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (whitespaceSplit.Length >= 2)
{
productSegment = string.Join(' ', whitespaceSplit[..^1]);
versionSegment = whitespaceSplit[^1];
return true;
}
productSegment = string.Empty;
versionSegment = string.Empty;
return false;
}
private static List<string> ExtractProducts(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var normalized = segment.Replace('\t', ' ').Trim();
var tokens = normalized
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
.Select(static token => token.Trim())
.Where(static token => token.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return tokens;
}
private static List<string> ExtractVersions(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var matches = VersionTokenRegex.Matches(segment);
if (matches.Count == 0)
{
return new List<string>();
}
var versions = new List<string>(matches.Count);
foreach (Match match in matches)
{
if (match.Groups.Count == 0)
{
continue;
}
var value = match.Groups[1].Value.Trim();
if (value.Length == 0)
{
continue;
}
versions.Add(value);
}
return versions
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(32)
.ToList();
}
}
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
{
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
{
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
}
public int GetHashCode(CertCcVendorPatch obj)
{
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
return HashCode.Combine(product, version);
}
}
}

View File

@@ -1,22 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertCc;
internal static class CertCcJobKinds
{
public const string Fetch = "source:cert-cc:fetch";
}
internal sealed class CertCcFetchJob : IJob
{
private readonly CertCcConnector _connector;
public CertCcFetchJob(CertCcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertCc;
internal static class CertCcJobKinds
{
public const string Fetch = "source:cert-cc:fetch";
}
internal sealed class CertCcFetchJob : IJob
{
private readonly CertCcConnector _connector;
public CertCcFetchJob(CertCcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]

View File

@@ -4,7 +4,7 @@ using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.CertFr.Configuration;
using StellaOps.Concelier.Connector.CertFr.Internal;
using StellaOps.Concelier.Connector.Common;
@@ -236,7 +236,7 @@ public sealed class CertFrConnector : IFeedConnector
}
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = BsonDocument.Parse(json);
var payload = DocumentObject.Parse(json);
var validatedAt = _timeProvider.GetUtcNow();
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
@@ -332,6 +332,6 @@ public sealed class CertFrConnector : IFeedConnector
private async Task UpdateCursorAsync(CertFrCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-fr";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertFrConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertFrConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-fr";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertFrConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertFrConnector>();
}
}

View File

@@ -1,54 +1,54 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-fr";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertFrConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertFrFetchJob>();
services.AddTransient<CertFrParseJob>();
services.AddTransient<CertFrMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob));
EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob));
EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-fr";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertFrConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertFrFetchJob>();
services.AddTransient<CertFrParseJob>();
services.AddTransient<CertFrMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob));
EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob));
EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -1,36 +1,36 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
using StellaOps.Concelier.Connector.CertFr.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertFr;
public static class CertFrServiceCollectionExtensions
{
public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action<CertFrOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertFrOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value;
clientOptions.BaseAddress = options.FeedUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
});
services.TryAddSingleton<CertFrFeedClient>();
services.AddTransient<CertFrConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
using StellaOps.Concelier.Connector.CertFr.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertFr;
public static class CertFrServiceCollectionExtensions
{
public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action<CertFrOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertFrOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value;
clientOptions.BaseAddress = options.FeedUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
});
services.TryAddSingleton<CertFrFeedClient>();
services.AddTransient<CertFrConnector>();
return services;
}
}

View File

@@ -1,46 +1,46 @@
using System;
namespace StellaOps.Concelier.Connector.CertFr.Configuration;
public sealed class CertFrOptions
{
public const string HttpClientName = "cert-fr";
public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
public int MaxItemsPerFetch { get; set; } = 100;
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("InitialBackfill must be a positive duration.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (MaxItemsPerFetch <= 0)
{
throw new InvalidOperationException("MaxItemsPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}
using System;
namespace StellaOps.Concelier.Connector.CertFr.Configuration;
public sealed class CertFrOptions
{
public const string HttpClientName = "cert-fr";
public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
public int MaxItemsPerFetch { get; set; } = 100;
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("InitialBackfill must be a positive duration.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (MaxItemsPerFetch <= 0)
{
throw new InvalidOperationException("MaxItemsPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}

View File

@@ -1,88 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertFrCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertFrCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var value)
? ParseDate(value)
: null;
return new CertFrCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertFrCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array)
{
return Array.Empty<Guid>();
}
var result = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
result.Add(guid);
}
}
return result;
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertFrCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertFrCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var value)
? ParseDate(value)
: null;
return new CertFrCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertFrCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var raw) || raw is not DocumentArray array)
{
return Array.Empty<Guid>();
}
var result = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
result.Add(guid);
}
}
return result;
}
}

View File

@@ -1,77 +1,77 @@
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDocumentMetadata(
string AdvisoryId,
string Title,
DateTimeOffset Published,
Uri DetailUri,
string? Summary)
{
private const string AdvisoryIdKey = "certfr.advisoryId";
private const string TitleKey = "certfr.title";
private const string PublishedKey = "certfr.published";
private const string SummaryKey = "certfr.summary";
public static CertFrDocumentMetadata FromDocument(DocumentRecord document)
{
ArgumentNullException.ThrowIfNull(document);
if (document.Metadata is null)
{
throw new InvalidOperationException("Cert-FR document metadata is missing.");
}
var metadata = document.Metadata;
if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId))
{
throw new InvalidOperationException("Cert-FR advisory id metadata missing.");
}
if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title))
{
throw new InvalidOperationException("Cert-FR title metadata missing.");
}
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
{
throw new InvalidOperationException("Cert-FR published metadata invalid.");
}
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
{
throw new InvalidOperationException("Cert-FR document URI invalid.");
}
metadata.TryGetValue(SummaryKey, out var summary);
return new CertFrDocumentMetadata(
advisoryId.Trim(),
title.Trim(),
published.ToUniversalTime(),
detailUri,
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
}
public static IReadOnlyDictionary<string, string> CreateMetadata(CertFrFeedItem item)
{
ArgumentNullException.ThrowIfNull(item);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
[AdvisoryIdKey] = item.AdvisoryId,
[TitleKey] = item.Title ?? item.AdvisoryId,
[PublishedKey] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Summary))
{
metadata[SummaryKey] = item.Summary!;
}
return metadata;
}
}
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDocumentMetadata(
string AdvisoryId,
string Title,
DateTimeOffset Published,
Uri DetailUri,
string? Summary)
{
private const string AdvisoryIdKey = "certfr.advisoryId";
private const string TitleKey = "certfr.title";
private const string PublishedKey = "certfr.published";
private const string SummaryKey = "certfr.summary";
public static CertFrDocumentMetadata FromDocument(DocumentRecord document)
{
ArgumentNullException.ThrowIfNull(document);
if (document.Metadata is null)
{
throw new InvalidOperationException("Cert-FR document metadata is missing.");
}
var metadata = document.Metadata;
if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId))
{
throw new InvalidOperationException("Cert-FR advisory id metadata missing.");
}
if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title))
{
throw new InvalidOperationException("Cert-FR title metadata missing.");
}
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
{
throw new InvalidOperationException("Cert-FR published metadata invalid.");
}
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
{
throw new InvalidOperationException("Cert-FR document URI invalid.");
}
metadata.TryGetValue(SummaryKey, out var summary);
return new CertFrDocumentMetadata(
advisoryId.Trim(),
title.Trim(),
published.ToUniversalTime(),
detailUri,
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
}
public static IReadOnlyDictionary<string, string> CreateMetadata(CertFrFeedItem item)
{
ArgumentNullException.ThrowIfNull(item);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
[AdvisoryIdKey] = item.AdvisoryId,
[TitleKey] = item.Title ?? item.AdvisoryId,
[PublishedKey] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Summary))
{
metadata[SummaryKey] = item.Summary!;
}
return metadata;
}
}

View File

@@ -1,14 +1,14 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDto(
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("detailUrl")] string DetailUrl,
[property: JsonPropertyName("published")] DateTimeOffset Published,
[property: JsonPropertyName("summary")] string? Summary,
[property: JsonPropertyName("content")] string Content,
[property: JsonPropertyName("references")] IReadOnlyList<string> References);
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDto(
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("detailUrl")] string DetailUrl,
[property: JsonPropertyName("published")] DateTimeOffset Published,
[property: JsonPropertyName("summary")] string? Summary,
[property: JsonPropertyName("content")] string Content,
[property: JsonPropertyName("references")] IReadOnlyList<string> References);

View File

@@ -1,109 +1,109 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed class CertFrFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertFrOptions _options;
private readonly ILogger<CertFrFeedClient> _logger;
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertFrFeedItem>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertFrFeedItem>();
var now = DateTimeOffset.UtcNow;
foreach (var itemElement in document.Descendants("item"))
{
var link = itemElement.Element("link")?.Value;
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
{
continue;
}
var title = itemElement.Element("title")?.Value?.Trim();
var summary = itemElement.Element("description")?.Value?.Trim();
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
if (published < windowStart)
{
continue;
}
if (published > windowEnd)
{
published = windowEnd;
}
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
}
return items
.OrderBy(item => item.Published)
.Take(_options.MaxItemsPerFetch)
.ToArray();
}
private static DateTimeOffset? ParsePublished(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed;
}
return null;
}
private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri)
{
var guid = itemElement.Element("guid")?.Value;
if (!string.IsNullOrWhiteSpace(guid))
{
return guid.Trim();
}
var segments = detailUri.Segments;
if (segments.Length > 0)
{
var slug = segments[^1].Trim('/');
if (!string.IsNullOrWhiteSpace(slug))
{
return slug;
}
}
return detailUri.AbsoluteUri;
}
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed class CertFrFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertFrOptions _options;
private readonly ILogger<CertFrFeedClient> _logger;
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertFrFeedItem>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertFrFeedItem>();
var now = DateTimeOffset.UtcNow;
foreach (var itemElement in document.Descendants("item"))
{
var link = itemElement.Element("link")?.Value;
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
{
continue;
}
var title = itemElement.Element("title")?.Value?.Trim();
var summary = itemElement.Element("description")?.Value?.Trim();
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
if (published < windowStart)
{
continue;
}
if (published > windowEnd)
{
published = windowEnd;
}
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
}
return items
.OrderBy(item => item.Published)
.Take(_options.MaxItemsPerFetch)
.ToArray();
}
private static DateTimeOffset? ParsePublished(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed;
}
return null;
}
private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri)
{
var guid = itemElement.Element("guid")?.Value;
if (!string.IsNullOrWhiteSpace(guid))
{
return guid.Trim();
}
var segments = detailUri.Segments;
if (segments.Length > 0)
{
var slug = segments[^1].Trim('/');
if (!string.IsNullOrWhiteSpace(slug))
{
return slug;
}
}
return detailUri.AbsoluteUri;
}
}

View File

@@ -1,10 +1,10 @@
using System;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed record CertFrFeedItem(
string AdvisoryId,
Uri DetailUri,
DateTimeOffset Published,
string? Title,
string? Summary);
using System;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed record CertFrFeedItem(
string AdvisoryId,
Uri DetailUri,
DateTimeOffset Published,
string? Title,
string? Summary);

View File

@@ -1,25 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrMapper
{
public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
var advisoryKey = $"cert-fr/{dto.AdvisoryId}";
var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime());
var aliases = new List<string>
{
$"CERT-FR:{dto.AdvisoryId}",
};
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrMapper
{
public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
var advisoryKey = $"cert-fr/{dto.AdvisoryId}";
var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime());
var aliases = new List<string>
{
$"CERT-FR:{dto.AdvisoryId}",
};
var references = BuildReferences(dto, provenance).ToArray();
var affectedPackages = BuildAffectedPackages(dto, provenance).ToArray();
@@ -45,22 +45,22 @@ internal static class CertFrMapper
var comparer = StringComparer.OrdinalIgnoreCase;
var entries = new List<(AdvisoryReference Reference, int Priority)>
{
(new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0),
};
foreach (var url in dto.References)
{
entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1));
}
return entries
.GroupBy(tuple => tuple.Reference.Url, comparer)
.Select(group => group
.OrderBy(t => t.Priority)
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
.ThenBy(t => t.Reference.Url, comparer)
.First())
.OrderBy(t => t.Priority)
(new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0),
};
foreach (var url in dto.References)
{
entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1));
}
return entries
.GroupBy(tuple => tuple.Reference.Url, comparer)
.Select(group => group
.OrderBy(t => t.Priority)
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
.ThenBy(t => t.Reference.Url, comparer)
.First())
.OrderBy(t => t.Priority)
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
.ThenBy(t => t.Reference.Url, comparer)
.Select(t => t.Reference);

View File

@@ -1,80 +1,80 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrParser
{
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata)
{
ArgumentException.ThrowIfNullOrEmpty(html);
ArgumentNullException.ThrowIfNull(metadata);
var sanitized = SanitizeHtml(html);
var summary = BuildSummary(metadata.Summary, sanitized);
var references = ExtractReferences(html);
return new CertFrDto(
metadata.AdvisoryId,
metadata.Title,
metadata.DetailUri.ToString(),
metadata.Published,
summary,
sanitized,
references);
}
private static string SanitizeHtml(string html)
{
var withoutScripts = ScriptRegex.Replace(html, string.Empty);
var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty);
var withoutTags = TagRegex.Replace(withoutStyles, " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
return WhitespaceRegex.Replace(decoded, " ").Trim();
}
private static string? BuildSummary(string? metadataSummary, string content)
{
if (!string.IsNullOrWhiteSpace(metadataSummary))
{
return metadataSummary.Trim();
}
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (sentences.Length > 0)
{
return sentences[0].Trim();
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static IReadOnlyList<string> ExtractReferences(string html)
{
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in AnchorRegex.Matches(html))
{
if (match.Success)
{
references.Add(match.Groups["url"].Value.Trim());
}
}
return references.Count == 0
? Array.Empty<string>()
: references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray();
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrParser
{
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata)
{
ArgumentException.ThrowIfNullOrEmpty(html);
ArgumentNullException.ThrowIfNull(metadata);
var sanitized = SanitizeHtml(html);
var summary = BuildSummary(metadata.Summary, sanitized);
var references = ExtractReferences(html);
return new CertFrDto(
metadata.AdvisoryId,
metadata.Title,
metadata.DetailUri.ToString(),
metadata.Published,
summary,
sanitized,
references);
}
private static string SanitizeHtml(string html)
{
var withoutScripts = ScriptRegex.Replace(html, string.Empty);
var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty);
var withoutTags = TagRegex.Replace(withoutStyles, " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
return WhitespaceRegex.Replace(decoded, " ").Trim();
}
private static string? BuildSummary(string? metadataSummary, string content)
{
if (!string.IsNullOrWhiteSpace(metadataSummary))
{
return metadataSummary.Trim();
}
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (sentences.Length > 0)
{
return sentences[0].Trim();
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static IReadOnlyList<string> ExtractReferences(string html)
{
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in AnchorRegex.Matches(html))
{
if (match.Success)
{
references.Add(match.Groups["url"].Value.Trim());
}
}
return references.Count == 0
? Array.Empty<string>()
: references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray();
}
}

View File

@@ -1,46 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertFr;
internal static class CertFrJobKinds
{
public const string Fetch = "source:cert-fr:fetch";
public const string Parse = "source:cert-fr:parse";
public const string Map = "source:cert-fr:map";
}
internal sealed class CertFrFetchJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrFetchJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertFrParseJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrParseJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertFrMapJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrMapJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertFr;
internal static class CertFrJobKinds
{
public const string Fetch = "source:cert-fr:fetch";
public const string Parse = "source:cert-fr:parse";
public const string Map = "source:cert-fr:map";
}
internal sealed class CertFrFetchJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrFetchJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertFrParseJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrParseJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertFrMapJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrMapJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

View File

@@ -6,7 +6,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
@@ -226,7 +226,7 @@ public sealed class CertInConnector : IFeedConnector
}
var dto = CertInDetailParser.Parse(listing, rawBytes);
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var payload = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certin.v1", payload, _timeProvider.GetUtcNow());
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
@@ -271,9 +271,9 @@ public sealed class CertInConnector : IFeedConnector
continue;
}
var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Bson.IO.JsonWriterSettings
var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Documents.IO.JsonWriterSettings
{
OutputMode = StellaOps.Concelier.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
OutputMode = StellaOps.Concelier.Documents.IO.JsonOutputMode.RelaxedExtendedJson,
});
CertInAdvisoryDto dto;
@@ -423,7 +423,7 @@ public sealed class CertInConnector : IFeedConnector
private Task UpdateCursorAsync(CertInCursor cursor, CancellationToken cancellationToken)
{
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken);
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), _timeProvider.GetUtcNow(), cancellationToken);
}
private static bool TryDeserializeListing(IReadOnlyDictionary<string, string>? metadata, out CertInListingItem listing)

View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-in";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<CertInConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-in";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<CertInConnector>(services);
}
}

View File

@@ -1,54 +1,54 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-in";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertInConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertInFetchJob>();
services.AddTransient<CertInParseJob>();
services.AddTransient<CertInMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob));
EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob));
EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-in";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertInConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertInFetchJob>();
services.AddTransient<CertInParseJob>();
services.AddTransient<CertInMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob));
EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob));
EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -1,37 +1,37 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertIn;
public static class CertInServiceCollectionExtensions
{
public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action<CertInOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertInOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value;
clientOptions.BaseAddress = options.AlertsEndpoint;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host);
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddTransient<CertInClient>();
services.AddTransient<CertInConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertIn;
public static class CertInServiceCollectionExtensions
{
public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action<CertInOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertInOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value;
clientOptions.BaseAddress = options.AlertsEndpoint;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host);
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddTransient<CertInClient>();
services.AddTransient<CertInConnector>();
return services;
}
}

View File

@@ -1,68 +1,68 @@
using System;
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Concelier.Connector.CertIn.Configuration;
public sealed class CertInOptions
{
public static string HttpClientName => "source.certin";
/// <summary>
/// Endpoint returning a paginated list of recent advisories.
/// </summary>
public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute);
/// <summary>
/// Size of the rolling fetch window.
/// </summary>
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
/// <summary>
/// Overlap applied to subsequent windows.
/// </summary>
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
/// <summary>
/// Maximum pages fetched per cycle.
/// </summary>
public int MaxPagesPerFetch { get; set; } = 5;
/// <summary>
/// Delay between successive HTTP requests.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
[MemberNotNull(nameof(AlertsEndpoint))]
public void Validate()
{
if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("AlertsEndpoint must be an absolute URI.");
}
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("WindowSize must be greater than zero.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (WindowOverlap >= WindowSize)
{
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}
using System;
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Concelier.Connector.CertIn.Configuration;
public sealed class CertInOptions
{
public static string HttpClientName => "source.certin";
/// <summary>
/// Endpoint returning a paginated list of recent advisories.
/// </summary>
public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute);
/// <summary>
/// Size of the rolling fetch window.
/// </summary>
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
/// <summary>
/// Overlap applied to subsequent windows.
/// </summary>
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
/// <summary>
/// Maximum pages fetched per cycle.
/// </summary>
public int MaxPagesPerFetch { get; set; } = 5;
/// <summary>
/// Delay between successive HTTP requests.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
[MemberNotNull(nameof(AlertsEndpoint))]
public void Validate()
{
if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("AlertsEndpoint must be an absolute URI.");
}
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("WindowSize must be greater than zero.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (WindowOverlap >= WindowSize)
{
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}

View File

@@ -1,16 +1,16 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInAdvisoryDto(
string AdvisoryId,
string Title,
string Link,
DateTimeOffset Published,
string? Summary,
string Content,
string? Severity,
ImmutableArray<string> CveIds,
ImmutableArray<string> VendorNames,
ImmutableArray<string> ReferenceLinks);
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInAdvisoryDto(
string AdvisoryId,
string Title,
string Link,
DateTimeOffset Published,
string? Summary,
string Content,
string? Severity,
ImmutableArray<string> CveIds,
ImmutableArray<string> VendorNames,
ImmutableArray<string> ReferenceLinks);

View File

@@ -1,129 +1,129 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed class CertInClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertInOptions _options;
private readonly ILogger<CertInClient> _logger;
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
{
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
{
continue;
}
items.Add(item);
}
return items;
}
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
{
item = null!;
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
{
return false;
}
var advisoryId = idElement.GetString();
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
? titleElement.GetString()
: advisoryId;
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
{
return false;
}
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
{
return false;
}
DateTimeOffset published;
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
{
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
{
return false;
}
}
else
{
return false;
}
string? summary = null;
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
{
summary = summaryElement.GetString();
}
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
return true;
}
private static Uri BuildPageUri(Uri baseUri, int page)
{
if (page <= 1)
{
return baseUri;
}
var builder = new UriBuilder(baseUri);
var trimmed = builder.Query.TrimStart('?');
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(trimmed)
? pageSegment
: $"{trimmed}&{pageSegment}";
return builder.Uri;
}
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed class CertInClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertInOptions _options;
private readonly ILogger<CertInClient> _logger;
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
{
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
{
continue;
}
items.Add(item);
}
return items;
}
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
{
item = null!;
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
{
return false;
}
var advisoryId = idElement.GetString();
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
? titleElement.GetString()
: advisoryId;
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
{
return false;
}
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
{
return false;
}
DateTimeOffset published;
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
{
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
{
return false;
}
}
else
{
return false;
}
string? summary = null;
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
{
summary = summaryElement.GetString();
}
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
return true;
}
private static Uri BuildPageUri(Uri baseUri, int page)
{
if (page <= 1)
{
return baseUri;
}
var builder = new UriBuilder(baseUri);
var trimmed = builder.Query.TrimStart('?');
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(trimmed)
? pageSegment
: $"{trimmed}&{pageSegment}";
return builder.Uri;
}
}

View File

@@ -1,88 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertInCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
? ParseDate(dateValue)
: null;
return new CertInCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertInCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
? ParseDate(dateValue)
: null;
return new CertInCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}

View File

@@ -1,187 +1,187 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal static class CertInDetailParser
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
{
ArgumentNullException.ThrowIfNull(listing);
var html = Encoding.UTF8.GetString(rawHtml);
var content = HtmlToPlainText(html);
var summary = listing.Summary ?? ExtractSummary(content);
var severity = ExtractSeverity(content);
var cves = ExtractCves(listing.Title, summary, content);
var vendors = ExtractVendors(summary, content);
var references = ExtractLinks(html);
return new CertInAdvisoryDto(
listing.AdvisoryId,
listing.Title,
listing.DetailUri.ToString(),
listing.Published,
summary,
content,
severity,
cves,
vendors,
references);
}
private static string HtmlToPlainText(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
return string.IsNullOrWhiteSpace(decoded)
? string.Empty
: Regex.Replace(decoded, "\\s+", " ").Trim();
}
private static string? ExtractSummary(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentenceTerminators = new[] { ".", "!", "?" };
foreach (var terminator in sentenceTerminators)
{
var index = content.IndexOf(terminator, StringComparison.Ordinal);
if (index > 0)
{
return content[..(index + terminator.Length)].Trim();
}
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static string? ExtractSeverity(string content)
{
var match = SeverityRegex.Match(content);
if (match.Success)
{
return match.Groups["value"].Value.Trim().ToLowerInvariant();
}
return null;
}
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Capture(string? text)
{
if (string.IsNullOrWhiteSpace(text))
{
return;
}
foreach (Match match in CveRegex.Matches(text))
{
if (match.Success)
{
set.Add(match.Value.ToUpperInvariant());
}
}
}
Capture(title);
Capture(summary);
Capture(content);
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
{
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Add(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var cleaned = value
.Replace("", "'", StringComparison.Ordinal)
.Trim();
if (cleaned.Length > 200)
{
cleaned = cleaned[..200];
}
if (!string.IsNullOrWhiteSpace(cleaned))
{
vendors.Add(cleaned);
}
}
if (!string.IsNullOrWhiteSpace(summary))
{
foreach (Match match in VendorRegex.Matches(summary))
{
Add(match.Groups["value"].Value);
}
}
foreach (Match match in VendorRegex.Matches(content))
{
Add(match.Groups["value"].Value);
}
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
{
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
Add(fallback);
}
return vendors.Count == 0
? ImmutableArray<string>.Empty
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractLinks(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ImmutableArray<string>.Empty;
}
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in LinkRegex.Matches(html))
{
if (match.Success)
{
links.Add(match.Groups[1].Value);
}
}
return links.Count == 0
? ImmutableArray<string>.Empty
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
}
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal static class CertInDetailParser
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
{
ArgumentNullException.ThrowIfNull(listing);
var html = Encoding.UTF8.GetString(rawHtml);
var content = HtmlToPlainText(html);
var summary = listing.Summary ?? ExtractSummary(content);
var severity = ExtractSeverity(content);
var cves = ExtractCves(listing.Title, summary, content);
var vendors = ExtractVendors(summary, content);
var references = ExtractLinks(html);
return new CertInAdvisoryDto(
listing.AdvisoryId,
listing.Title,
listing.DetailUri.ToString(),
listing.Published,
summary,
content,
severity,
cves,
vendors,
references);
}
private static string HtmlToPlainText(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
return string.IsNullOrWhiteSpace(decoded)
? string.Empty
: Regex.Replace(decoded, "\\s+", " ").Trim();
}
private static string? ExtractSummary(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentenceTerminators = new[] { ".", "!", "?" };
foreach (var terminator in sentenceTerminators)
{
var index = content.IndexOf(terminator, StringComparison.Ordinal);
if (index > 0)
{
return content[..(index + terminator.Length)].Trim();
}
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static string? ExtractSeverity(string content)
{
var match = SeverityRegex.Match(content);
if (match.Success)
{
return match.Groups["value"].Value.Trim().ToLowerInvariant();
}
return null;
}
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Capture(string? text)
{
if (string.IsNullOrWhiteSpace(text))
{
return;
}
foreach (Match match in CveRegex.Matches(text))
{
if (match.Success)
{
set.Add(match.Value.ToUpperInvariant());
}
}
}
Capture(title);
Capture(summary);
Capture(content);
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
{
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Add(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var cleaned = value
.Replace("", "'", StringComparison.Ordinal)
.Trim();
if (cleaned.Length > 200)
{
cleaned = cleaned[..200];
}
if (!string.IsNullOrWhiteSpace(cleaned))
{
vendors.Add(cleaned);
}
}
if (!string.IsNullOrWhiteSpace(summary))
{
foreach (Match match in VendorRegex.Matches(summary))
{
Add(match.Groups["value"].Value);
}
}
foreach (Match match in VendorRegex.Matches(content))
{
Add(match.Groups["value"].Value);
}
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
{
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
Add(fallback);
}
return vendors.Count == 0
? ImmutableArray<string>.Empty
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractLinks(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ImmutableArray<string>.Empty;
}
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in LinkRegex.Matches(html))
{
if (match.Success)
{
links.Add(match.Groups[1].Value);
}
}
return links.Count == 0
? ImmutableArray<string>.Empty
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
}

View File

@@ -1,10 +1,10 @@
using System;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed record CertInListingItem(
string AdvisoryId,
string Title,
Uri DetailUri,
DateTimeOffset Published,
string? Summary);
using System;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed record CertInListingItem(
string AdvisoryId,
string Title,
Uri DetailUri,
DateTimeOffset Published,
string? Summary);

View File

@@ -1,46 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertIn;
internal static class CertInJobKinds
{
public const string Fetch = "source:cert-in:fetch";
public const string Parse = "source:cert-in:parse";
public const string Map = "source:cert-in:map";
}
internal sealed class CertInFetchJob : IJob
{
private readonly CertInConnector _connector;
public CertInFetchJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertInParseJob : IJob
{
private readonly CertInConnector _connector;
public CertInParseJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertInMapJob : IJob
{
private readonly CertInConnector _connector;
public CertInMapJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertIn;
internal static class CertInJobKinds
{
public const string Fetch = "source:cert-in:fetch";
public const string Parse = "source:cert-in:parse";
public const string Map = "source:cert-in:map";
}
internal sealed class CertInFetchJob : IJob
{
private readonly CertInConnector _connector;
public CertInFetchJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertInParseJob : IJob
{
private readonly CertInConnector _connector;
public CertInParseJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertInMapJob : IJob
{
private readonly CertInConnector _connector;
public CertInMapJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

View File

@@ -1,29 +1,29 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
/// </summary>
public static class PaginationPlanner
{
/// <summary>
/// Enumerates additional page start indices given the total result count returned by the source.
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
/// </summary>
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
{
if (totalResults <= 0 || resultsPerPage <= 0)
{
yield break;
}
if (firstPageStartIndex < 0)
{
firstPageStartIndex = 0;
}
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
{
yield return start;
}
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
/// </summary>
public static class PaginationPlanner
{
/// <summary>
/// Enumerates additional page start indices given the total result count returned by the source.
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
/// </summary>
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
{
if (totalResults <= 0 || resultsPerPage <= 0)
{
yield break;
}
if (firstPageStartIndex < 0)
{
firstPageStartIndex = 0;
}
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
{
yield return start;
}
}
}

View File

@@ -1,43 +1,43 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Configuration applied when advancing sliding time-window cursors.
/// </summary>
public sealed class TimeWindowCursorOptions
{
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
public void EnsureValid()
{
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Window overlap cannot be negative.");
}
if (Overlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill must be positive.");
}
if (MinimumWindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Minimum window size must be positive.");
}
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Configuration applied when advancing sliding time-window cursors.
/// </summary>
public sealed class TimeWindowCursorOptions
{
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
public void EnsureValid()
{
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Window overlap cannot be negative.");
}
if (Overlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill must be positive.");
}
if (MinimumWindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Minimum window size must be positive.");
}
}
}

View File

@@ -1,50 +1,50 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Utility methods for computing sliding time-window ranges used by connectors.
/// </summary>
public static class TimeWindowCursorPlanner
{
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
{
ArgumentNullException.ThrowIfNull(options);
options.EnsureValid();
var effectiveState = state ?? TimeWindowCursorState.Empty;
var earliest = now - options.InitialBackfill;
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
if (anchorEnd < earliest)
{
anchorEnd = earliest;
}
var start = anchorEnd - options.Overlap;
if (start < earliest)
{
start = earliest;
}
var end = start + options.WindowSize;
if (end > now)
{
end = now;
}
if (end <= start)
{
end = start + options.MinimumWindowSize;
if (end > now)
{
end = now;
}
}
if (end <= start)
{
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
}
return new TimeWindow(start, end);
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Utility methods for computing sliding time-window ranges used by connectors.
/// </summary>
public static class TimeWindowCursorPlanner
{
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
{
ArgumentNullException.ThrowIfNull(options);
options.EnsureValid();
var effectiveState = state ?? TimeWindowCursorState.Empty;
var earliest = now - options.InitialBackfill;
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
if (anchorEnd < earliest)
{
anchorEnd = earliest;
}
var start = anchorEnd - options.Overlap;
if (start < earliest)
{
start = earliest;
}
var end = start + options.WindowSize;
if (end > now)
{
end = now;
}
if (end <= start)
{
end = start + options.MinimumWindowSize;
if (end > now)
{
end = now;
}
}
if (end <= start)
{
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
}
return new TimeWindow(start, end);
}
}

View File

@@ -1,84 +1,84 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Represents the persisted state of a sliding time-window cursor.
/// </summary>
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
{
public static TimeWindowCursorState Empty { get; } = new(null, null);
public TimeWindowCursorState WithWindow(TimeWindow window)
{
return new TimeWindowCursorState(window.Start, window.End);
}
public BsonDocument ToBsonDocument(string startField = "windowStart", string endField = "windowEnd")
{
var document = new BsonDocument();
WriteTo(document, startField, endField);
return document;
}
public void WriteTo(BsonDocument document, string startField = "windowStart", string endField = "windowEnd")
{
ArgumentNullException.ThrowIfNull(document);
ArgumentException.ThrowIfNullOrEmpty(startField);
ArgumentException.ThrowIfNullOrEmpty(endField);
document.Remove(startField);
document.Remove(endField);
if (LastWindowStart.HasValue)
{
document[startField] = LastWindowStart.Value.UtcDateTime;
}
if (LastWindowEnd.HasValue)
{
document[endField] = LastWindowEnd.Value.UtcDateTime;
}
}
public static TimeWindowCursorState FromBsonDocument(BsonDocument? document, string startField = "windowStart", string endField = "windowEnd")
{
if (document is null)
{
return Empty;
}
DateTimeOffset? start = null;
DateTimeOffset? end = null;
if (document.TryGetValue(startField, out var startValue))
{
start = ReadDateTimeOffset(startValue);
}
if (document.TryGetValue(endField, out var endValue))
{
end = ReadDateTimeOffset(endValue);
}
return new TimeWindowCursorState(start, end);
}
private static DateTimeOffset? ReadDateTimeOffset(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
/// <summary>
/// Simple value object describing a time window.
/// </summary>
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
{
public TimeSpan Duration => End - Start;
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Represents the persisted state of a sliding time-window cursor.
/// </summary>
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
{
public static TimeWindowCursorState Empty { get; } = new(null, null);
public TimeWindowCursorState WithWindow(TimeWindow window)
{
return new TimeWindowCursorState(window.Start, window.End);
}
public DocumentObject ToDocumentObject(string startField = "windowStart", string endField = "windowEnd")
{
var document = new DocumentObject();
WriteTo(document, startField, endField);
return document;
}
public void WriteTo(DocumentObject document, string startField = "windowStart", string endField = "windowEnd")
{
ArgumentNullException.ThrowIfNull(document);
ArgumentException.ThrowIfNullOrEmpty(startField);
ArgumentException.ThrowIfNullOrEmpty(endField);
document.Remove(startField);
document.Remove(endField);
if (LastWindowStart.HasValue)
{
document[startField] = LastWindowStart.Value.UtcDateTime;
}
if (LastWindowEnd.HasValue)
{
document[endField] = LastWindowEnd.Value.UtcDateTime;
}
}
public static TimeWindowCursorState FromDocumentObject(DocumentObject? document, string startField = "windowStart", string endField = "windowEnd")
{
if (document is null)
{
return Empty;
}
DateTimeOffset? start = null;
DateTimeOffset? end = null;
if (document.TryGetValue(startField, out var startValue))
{
start = ReadDateTimeOffset(startValue);
}
if (document.TryGetValue(endField, out var endValue))
{
end = ReadDateTimeOffset(endValue);
}
return new TimeWindowCursorState(start, end);
}
private static DateTimeOffset? ReadDateTimeOffset(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
/// <summary>
/// Simple value object describing a time window.
/// </summary>
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
{
public TimeSpan Duration => End - Start;
}

View File

@@ -1,27 +1,27 @@
namespace StellaOps.Concelier.Connector.Common;
/// <summary>
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
/// </summary>
public static class DocumentStatuses
{
/// <summary>
/// Document captured from the upstream source and awaiting schema validation/parsing.
/// </summary>
public const string PendingParse = "pending-parse";
/// <summary>
/// Document parsed and sanitized; awaiting canonical mapping.
/// </summary>
public const string PendingMap = "pending-map";
/// <summary>
/// Document fully mapped to canonical advisories.
/// </summary>
public const string Mapped = "mapped";
/// <summary>
/// Document failed processing; requires manual intervention before retry.
/// </summary>
public const string Failed = "failed";
}
namespace StellaOps.Concelier.Connector.Common;
/// <summary>
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
/// </summary>
public static class DocumentStatuses
{
/// <summary>
/// Document captured from the upstream source and awaiting schema validation/parsing.
/// </summary>
public const string PendingParse = "pending-parse";
/// <summary>
/// Document parsed and sanitized; awaiting canonical mapping.
/// </summary>
public const string PendingMap = "pending-map";
/// <summary>
/// Document fully mapped to canonical advisories.
/// </summary>
public const string Mapped = "mapped";
/// <summary>
/// Document failed processing; requires manual intervention before retry.
/// </summary>
public const string Failed = "failed";
}

View File

@@ -1,43 +1,43 @@
using System.Security.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
/// </summary>
public sealed class CryptoJitterSource : IJitterSource
{
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
{
if (maxInclusive < minInclusive)
{
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
}
if (minInclusive < TimeSpan.Zero)
{
minInclusive = TimeSpan.Zero;
}
if (maxInclusive == minInclusive)
{
return minInclusive;
}
var minTicks = minInclusive.Ticks;
var maxTicks = maxInclusive.Ticks;
var range = maxTicks - minTicks;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer);
var ratio = sample / (double)ulong.MaxValue;
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
if (jitterTicks > range)
{
jitterTicks = range;
}
return TimeSpan.FromTicks(minTicks + jitterTicks);
}
}
using System.Security.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
/// </summary>
public sealed class CryptoJitterSource : IJitterSource
{
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
{
if (maxInclusive < minInclusive)
{
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
}
if (minInclusive < TimeSpan.Zero)
{
minInclusive = TimeSpan.Zero;
}
if (maxInclusive == minInclusive)
{
return minInclusive;
}
var minTicks = minInclusive.Ticks;
var maxTicks = maxInclusive.Ticks;
var range = maxTicks - minTicks;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer);
var ratio = sample / (double)ulong.MaxValue;
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
if (jitterTicks > range)
{
jitterTicks = range;
}
return TimeSpan.FromTicks(minTicks + jitterTicks);
}
}

View File

@@ -1,9 +1,9 @@
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Produces random jitter durations used to decorrelate retries.
/// </summary>
public interface IJitterSource
{
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
}
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Produces random jitter durations used to decorrelate retries.
/// </summary>
public interface IJitterSource
{
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
}

View File

@@ -1,18 +1,18 @@
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Result of fetching raw response content without persisting a document.
/// </summary>
public sealed record SourceFetchContentResult
{
private SourceFetchContentResult(
HttpStatusCode statusCode,
byte[]? content,
bool notModified,
string? etag,
DateTimeOffset? lastModified,
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Result of fetching raw response content without persisting a document.
/// </summary>
public sealed record SourceFetchContentResult
{
private SourceFetchContentResult(
HttpStatusCode statusCode,
byte[]? content,
bool notModified,
string? etag,
DateTimeOffset? lastModified,
string? contentType,
int attempts,
IReadOnlyDictionary<string, string>? headers)
@@ -30,14 +30,14 @@ public sealed record SourceFetchContentResult
public HttpStatusCode StatusCode { get; }
public byte[]? Content { get; }
public bool IsSuccess => Content is not null;
public bool IsNotModified { get; }
public string? ETag { get; }
public DateTimeOffset? LastModified { get; }
public bool IsSuccess => Content is not null;
public bool IsNotModified { get; }
public string? ETag { get; }
public DateTimeOffset? LastModified { get; }
public string? ContentType { get; }

View File

@@ -1,24 +1,24 @@
using System.Collections.Generic;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Parameters describing a fetch operation for a source connector.
/// </summary>
public sealed record SourceFetchRequest(
string ClientName,
string SourceName,
HttpMethod Method,
Uri RequestUri,
IReadOnlyDictionary<string, string>? Metadata = null,
string? ETag = null,
DateTimeOffset? LastModified = null,
TimeSpan? TimeoutOverride = null,
IReadOnlyList<string>? AcceptHeaders = null)
{
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
: this(clientName, sourceName, HttpMethod.Get, requestUri)
{
}
}
using System.Collections.Generic;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Parameters describing a fetch operation for a source connector.
/// </summary>
public sealed record SourceFetchRequest(
string ClientName,
string SourceName,
HttpMethod Method,
Uri RequestUri,
IReadOnlyDictionary<string, string>? Metadata = null,
string? ETag = null,
DateTimeOffset? LastModified = null,
TimeSpan? TimeoutOverride = null,
IReadOnlyList<string>? AcceptHeaders = null)
{
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
: this(clientName, sourceName, HttpMethod.Get, requestUri)
{
}
}

View File

@@ -1,34 +1,34 @@
using System.Net;
using StellaOps.Concelier.Storage.Contracts;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Outcome of fetching a raw document from an upstream source.
/// </summary>
public sealed record SourceFetchResult
{
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
{
StatusCode = statusCode;
Document = document;
IsNotModified = notModified;
}
public HttpStatusCode StatusCode { get; }
public StorageDocument? Document { get; }
public bool IsSuccess => Document is not null;
public bool IsNotModified { get; }
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
=> new(statusCode, document, notModified: false);
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: true);
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: false);
}
using System.Net;
using StellaOps.Concelier.Storage.Contracts;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Outcome of fetching a raw document from an upstream source.
/// </summary>
public sealed record SourceFetchResult
{
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
{
StatusCode = statusCode;
Document = document;
IsNotModified = notModified;
}
public HttpStatusCode StatusCode { get; }
public StorageDocument? Document { get; }
public bool IsSuccess => Document is not null;
public bool IsNotModified { get; }
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
=> new(statusCode, document, notModified: false);
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: true);
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: false);
}

View File

@@ -2,10 +2,10 @@ using System.Globalization;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Provides retry/backoff behavior for source HTTP fetches.
/// </summary>
/// <summary>
/// Provides retry/backoff behavior for source HTTP fetches.
/// </summary>
internal static class SourceRetryPolicy
{
private static readonly StringComparer HeaderComparer = StringComparer.OrdinalIgnoreCase;
@@ -15,34 +15,34 @@ internal static class SourceRetryPolicy
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> sender,
int maxAttempts,
TimeSpan baseDelay,
IJitterSource jitterSource,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
var attempt = 0;
while (true)
{
attempt++;
using var request = requestFactory();
HttpResponseMessage response;
try
{
response = await sender(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (attempt < maxAttempts)
{
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
IJitterSource jitterSource,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
var attempt = 0;
while (true)
{
attempt++;
using var request = requestFactory();
HttpResponseMessage response;
try
{
response = await sender(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (attempt < maxAttempts)
{
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
if (NeedsRetry(response) && attempt < maxAttempts)
{
var delay = ComputeDelay(
@@ -55,11 +55,11 @@ internal static class SourceRetryPolicy
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
return response;
}
}
return response;
}
}
private static bool NeedsRetry(HttpResponseMessage response)
{
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
@@ -76,13 +76,13 @@ internal static class SourceRetryPolicy
return status >= 500 && status < 600;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
return retryAfter.Value;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
return retryAfter.Value;
}
var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1));
var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250))
?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250));

Some files were not shown because too many files have changed in this diff Show More