up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,3 +1,3 @@
|
||||
using Xunit;
|
||||
|
||||
[assembly: CollectionBehavior(DisableTestParallelization = true)]
|
||||
using Xunit;
|
||||
|
||||
[assembly: CollectionBehavior(DisableTestParallelization = true)]
|
||||
|
||||
@@ -4,48 +4,48 @@ using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Concelier.Core.Attestation;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Contracts;
|
||||
|
||||
public sealed record AdvisoryIngestRequest(
|
||||
AdvisorySourceRequest Source,
|
||||
AdvisoryUpstreamRequest Upstream,
|
||||
AdvisoryContentRequest Content,
|
||||
AdvisoryIdentifiersRequest Identifiers,
|
||||
AdvisoryLinksetRequest? Linkset);
|
||||
|
||||
public sealed record AdvisorySourceRequest(
|
||||
[property: JsonPropertyName("vendor")] string Vendor,
|
||||
[property: JsonPropertyName("connector")] string Connector,
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("stream")] string? Stream);
|
||||
|
||||
public sealed record AdvisoryUpstreamRequest(
|
||||
[property: JsonPropertyName("upstreamId")] string UpstreamId,
|
||||
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("signature")] AdvisorySignatureRequest Signature,
|
||||
[property: JsonPropertyName("provenance")] IDictionary<string, string>? Provenance);
|
||||
|
||||
public sealed record AdvisorySignatureRequest(
|
||||
[property: JsonPropertyName("present")] bool Present,
|
||||
[property: JsonPropertyName("format")] string? Format,
|
||||
[property: JsonPropertyName("keyId")] string? KeyId,
|
||||
[property: JsonPropertyName("sig")] string? Signature,
|
||||
[property: JsonPropertyName("certificate")] string? Certificate,
|
||||
[property: JsonPropertyName("digest")] string? Digest);
|
||||
|
||||
public sealed record AdvisoryContentRequest(
|
||||
[property: JsonPropertyName("format")] string Format,
|
||||
[property: JsonPropertyName("specVersion")] string? SpecVersion,
|
||||
[property: JsonPropertyName("raw")] JsonElement Raw,
|
||||
[property: JsonPropertyName("encoding")] string? Encoding);
|
||||
|
||||
public sealed record AdvisoryIdentifiersRequest(
|
||||
[property: JsonPropertyName("primary")] string Primary,
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases);
|
||||
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Contracts;
|
||||
|
||||
public sealed record AdvisoryIngestRequest(
|
||||
AdvisorySourceRequest Source,
|
||||
AdvisoryUpstreamRequest Upstream,
|
||||
AdvisoryContentRequest Content,
|
||||
AdvisoryIdentifiersRequest Identifiers,
|
||||
AdvisoryLinksetRequest? Linkset);
|
||||
|
||||
public sealed record AdvisorySourceRequest(
|
||||
[property: JsonPropertyName("vendor")] string Vendor,
|
||||
[property: JsonPropertyName("connector")] string Connector,
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("stream")] string? Stream);
|
||||
|
||||
public sealed record AdvisoryUpstreamRequest(
|
||||
[property: JsonPropertyName("upstreamId")] string UpstreamId,
|
||||
[property: JsonPropertyName("documentVersion")] string? DocumentVersion,
|
||||
[property: JsonPropertyName("retrievedAt")] DateTimeOffset? RetrievedAt,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("signature")] AdvisorySignatureRequest Signature,
|
||||
[property: JsonPropertyName("provenance")] IDictionary<string, string>? Provenance);
|
||||
|
||||
public sealed record AdvisorySignatureRequest(
|
||||
[property: JsonPropertyName("present")] bool Present,
|
||||
[property: JsonPropertyName("format")] string? Format,
|
||||
[property: JsonPropertyName("keyId")] string? KeyId,
|
||||
[property: JsonPropertyName("sig")] string? Signature,
|
||||
[property: JsonPropertyName("certificate")] string? Certificate,
|
||||
[property: JsonPropertyName("digest")] string? Digest);
|
||||
|
||||
public sealed record AdvisoryContentRequest(
|
||||
[property: JsonPropertyName("format")] string Format,
|
||||
[property: JsonPropertyName("specVersion")] string? SpecVersion,
|
||||
[property: JsonPropertyName("raw")] JsonElement Raw,
|
||||
[property: JsonPropertyName("encoding")] string? Encoding);
|
||||
|
||||
public sealed record AdvisoryIdentifiersRequest(
|
||||
[property: JsonPropertyName("primary")] string Primary,
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases);
|
||||
|
||||
public sealed record AdvisoryLinksetRequest(
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string>? Aliases,
|
||||
[property: JsonPropertyName("scopes")] IReadOnlyList<string>? Scopes,
|
||||
@@ -66,23 +66,23 @@ public sealed record AdvisoryLinksetReferenceRequest(
|
||||
[property: JsonPropertyName("type")] string Type,
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("source")] string? Source);
|
||||
|
||||
public sealed record AdvisoryIngestResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("inserted")] bool Inserted,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
public sealed record AdvisoryRawRecordResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
|
||||
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
|
||||
|
||||
|
||||
public sealed record AdvisoryIngestResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("inserted")] bool Inserted,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
public sealed record AdvisoryRawRecordResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
|
||||
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
|
||||
|
||||
public sealed record AdvisoryRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
@@ -98,44 +98,44 @@ public sealed record AdvisoryRawProvenanceResponse(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
public sealed record AocVerifyRequest(
|
||||
[property: JsonPropertyName("since")] DateTimeOffset? Since,
|
||||
[property: JsonPropertyName("until")] DateTimeOffset? Until,
|
||||
[property: JsonPropertyName("limit")] int? Limit,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
|
||||
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
|
||||
|
||||
public sealed record AocVerifyResponse(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("window")] AocVerifyWindow Window,
|
||||
[property: JsonPropertyName("checked")] AocVerifyChecked Checked,
|
||||
[property: JsonPropertyName("violations")] IReadOnlyList<AocVerifyViolation> Violations,
|
||||
[property: JsonPropertyName("metrics")] AocVerifyMetrics Metrics,
|
||||
[property: JsonPropertyName("truncated")] bool Truncated);
|
||||
|
||||
public sealed record AocVerifyWindow(
|
||||
[property: JsonPropertyName("from")] DateTimeOffset From,
|
||||
[property: JsonPropertyName("to")] DateTimeOffset To);
|
||||
|
||||
public sealed record AocVerifyChecked(
|
||||
[property: JsonPropertyName("advisories")] int Advisories,
|
||||
[property: JsonPropertyName("vex")] int Vex);
|
||||
|
||||
public sealed record AocVerifyMetrics(
|
||||
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
|
||||
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
|
||||
|
||||
public sealed record AocVerifyViolation(
|
||||
[property: JsonPropertyName("code")] string Code,
|
||||
[property: JsonPropertyName("count")] int Count,
|
||||
[property: JsonPropertyName("examples")] IReadOnlyList<AocVerifyViolationExample> Examples);
|
||||
|
||||
public sealed record AocVerifyViolationExample(
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("documentId")] string DocumentId,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("path")] string Path);
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
public sealed record AocVerifyRequest(
|
||||
[property: JsonPropertyName("since")] DateTimeOffset? Since,
|
||||
[property: JsonPropertyName("until")] DateTimeOffset? Until,
|
||||
[property: JsonPropertyName("limit")] int? Limit,
|
||||
[property: JsonPropertyName("sources")] IReadOnlyList<string>? Sources,
|
||||
[property: JsonPropertyName("codes")] IReadOnlyList<string>? Codes);
|
||||
|
||||
public sealed record AocVerifyResponse(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("window")] AocVerifyWindow Window,
|
||||
[property: JsonPropertyName("checked")] AocVerifyChecked Checked,
|
||||
[property: JsonPropertyName("violations")] IReadOnlyList<AocVerifyViolation> Violations,
|
||||
[property: JsonPropertyName("metrics")] AocVerifyMetrics Metrics,
|
||||
[property: JsonPropertyName("truncated")] bool Truncated);
|
||||
|
||||
public sealed record AocVerifyWindow(
|
||||
[property: JsonPropertyName("from")] DateTimeOffset From,
|
||||
[property: JsonPropertyName("to")] DateTimeOffset To);
|
||||
|
||||
public sealed record AocVerifyChecked(
|
||||
[property: JsonPropertyName("advisories")] int Advisories,
|
||||
[property: JsonPropertyName("vex")] int Vex);
|
||||
|
||||
public sealed record AocVerifyMetrics(
|
||||
[property: JsonPropertyName("ingestion_write_total")] int IngestionWriteTotal,
|
||||
[property: JsonPropertyName("aoc_violation_total")] int AocViolationTotal);
|
||||
|
||||
public sealed record AocVerifyViolation(
|
||||
[property: JsonPropertyName("code")] string Code,
|
||||
[property: JsonPropertyName("count")] int Count,
|
||||
[property: JsonPropertyName("examples")] IReadOnlyList<AocVerifyViolationExample> Examples);
|
||||
|
||||
public sealed record AocVerifyViolationExample(
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("documentId")] string DocumentId,
|
||||
[property: JsonPropertyName("contentHash")] string ContentHash,
|
||||
[property: JsonPropertyName("path")] string Path);
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal sealed record StorageHealth(
|
||||
string Backend,
|
||||
bool Ready,
|
||||
DateTimeOffset? CheckedAt,
|
||||
double? LatencyMs,
|
||||
string? Error);
|
||||
|
||||
internal sealed record TelemetryHealth(
|
||||
bool Enabled,
|
||||
bool Tracing,
|
||||
bool Metrics,
|
||||
bool Logging);
|
||||
|
||||
internal sealed record HealthDocument(
|
||||
string Status,
|
||||
DateTimeOffset StartedAt,
|
||||
double UptimeSeconds,
|
||||
StorageHealth Storage,
|
||||
TelemetryHealth Telemetry);
|
||||
|
||||
internal sealed record ReadyDocument(
|
||||
string Status,
|
||||
DateTimeOffset StartedAt,
|
||||
double UptimeSeconds,
|
||||
StorageHealth Storage);
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal sealed record StorageHealth(
|
||||
string Backend,
|
||||
bool Ready,
|
||||
DateTimeOffset? CheckedAt,
|
||||
double? LatencyMs,
|
||||
string? Error);
|
||||
|
||||
internal sealed record TelemetryHealth(
|
||||
bool Enabled,
|
||||
bool Tracing,
|
||||
bool Metrics,
|
||||
bool Logging);
|
||||
|
||||
internal sealed record HealthDocument(
|
||||
string Status,
|
||||
DateTimeOffset StartedAt,
|
||||
double UptimeSeconds,
|
||||
StorageHealth Storage,
|
||||
TelemetryHealth Telemetry);
|
||||
|
||||
internal sealed record ReadyDocument(
|
||||
string Status,
|
||||
DateTimeOffset StartedAt,
|
||||
double UptimeSeconds,
|
||||
StorageHealth Storage);
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class JobMetrics
|
||||
{
|
||||
internal const string MeterName = "StellaOps.Concelier.WebService.Jobs";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName);
|
||||
|
||||
internal static readonly Counter<long> TriggerCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.triggered",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests accepted by the web service.");
|
||||
|
||||
internal static readonly Counter<long> TriggerConflictCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.trigger.conflict",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests that resulted in conflicts or rejections.");
|
||||
|
||||
internal static readonly Counter<long> TriggerFailureCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.trigger.failed",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests that failed at runtime.");
|
||||
}
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class JobMetrics
|
||||
{
|
||||
internal const string MeterName = "StellaOps.Concelier.WebService.Jobs";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName);
|
||||
|
||||
internal static readonly Counter<long> TriggerCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.triggered",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests accepted by the web service.");
|
||||
|
||||
internal static readonly Counter<long> TriggerConflictCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.trigger.conflict",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests that resulted in conflicts or rejections.");
|
||||
|
||||
internal static readonly Counter<long> TriggerFailureCounter = Meter.CreateCounter<long>(
|
||||
"web.jobs.trigger.failed",
|
||||
unit: "count",
|
||||
description: "Number of job trigger requests that failed at runtime.");
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class ProblemTypes
|
||||
{
|
||||
public const string NotFound = "https://stellaops.org/problems/not-found";
|
||||
public const string Validation = "https://stellaops.org/problems/validation";
|
||||
public const string Conflict = "https://stellaops.org/problems/conflict";
|
||||
public const string Locked = "https://stellaops.org/problems/locked";
|
||||
public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected";
|
||||
public const string JobFailure = "https://stellaops.org/problems/job-failure";
|
||||
public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable";
|
||||
}
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal static class ProblemTypes
|
||||
{
|
||||
public const string NotFound = "https://stellaops.org/problems/not-found";
|
||||
public const string Validation = "https://stellaops.org/problems/validation";
|
||||
public const string Conflict = "https://stellaops.org/problems/conflict";
|
||||
public const string Locked = "https://stellaops.org/problems/locked";
|
||||
public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected";
|
||||
public const string JobFailure = "https://stellaops.org/problems/job-failure";
|
||||
public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable";
|
||||
}
|
||||
|
||||
@@ -1,74 +1,74 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal sealed class ServiceStatus
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly DateTimeOffset _startedAt;
|
||||
private readonly object _sync = new();
|
||||
|
||||
private DateTimeOffset? _bootstrapCompletedAt;
|
||||
private TimeSpan? _bootstrapDuration;
|
||||
private DateTimeOffset? _lastReadyCheckAt;
|
||||
private TimeSpan? _lastStorageLatency;
|
||||
private string? _lastStorageError;
|
||||
private bool _lastReadySucceeded;
|
||||
|
||||
public ServiceStatus(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_startedAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
public ServiceHealthSnapshot CreateSnapshot()
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
return new ServiceHealthSnapshot(
|
||||
CapturedAt: _timeProvider.GetUtcNow(),
|
||||
StartedAt: _startedAt,
|
||||
BootstrapCompletedAt: _bootstrapCompletedAt,
|
||||
BootstrapDuration: _bootstrapDuration,
|
||||
LastReadyCheckAt: _lastReadyCheckAt,
|
||||
LastStorageLatency: _lastStorageLatency,
|
||||
LastStorageError: _lastStorageError,
|
||||
LastReadySucceeded: _lastReadySucceeded);
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkBootstrapCompleted(TimeSpan duration)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
_bootstrapCompletedAt = completedAt;
|
||||
_bootstrapDuration = duration;
|
||||
_lastReadySucceeded = true;
|
||||
_lastStorageLatency = duration;
|
||||
_lastStorageError = null;
|
||||
_lastReadyCheckAt = completedAt;
|
||||
}
|
||||
}
|
||||
|
||||
public void RecordStorageCheck(bool success, TimeSpan latency, string? error)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
_lastReadySucceeded = success;
|
||||
_lastStorageLatency = latency;
|
||||
_lastStorageError = success ? null : error;
|
||||
_lastReadyCheckAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record ServiceHealthSnapshot(
|
||||
DateTimeOffset CapturedAt,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset? BootstrapCompletedAt,
|
||||
TimeSpan? BootstrapDuration,
|
||||
DateTimeOffset? LastReadyCheckAt,
|
||||
TimeSpan? LastStorageLatency,
|
||||
string? LastStorageError,
|
||||
bool LastReadySucceeded);
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Diagnostics;
|
||||
|
||||
internal sealed class ServiceStatus
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly DateTimeOffset _startedAt;
|
||||
private readonly object _sync = new();
|
||||
|
||||
private DateTimeOffset? _bootstrapCompletedAt;
|
||||
private TimeSpan? _bootstrapDuration;
|
||||
private DateTimeOffset? _lastReadyCheckAt;
|
||||
private TimeSpan? _lastStorageLatency;
|
||||
private string? _lastStorageError;
|
||||
private bool _lastReadySucceeded;
|
||||
|
||||
public ServiceStatus(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_startedAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
public ServiceHealthSnapshot CreateSnapshot()
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
return new ServiceHealthSnapshot(
|
||||
CapturedAt: _timeProvider.GetUtcNow(),
|
||||
StartedAt: _startedAt,
|
||||
BootstrapCompletedAt: _bootstrapCompletedAt,
|
||||
BootstrapDuration: _bootstrapDuration,
|
||||
LastReadyCheckAt: _lastReadyCheckAt,
|
||||
LastStorageLatency: _lastStorageLatency,
|
||||
LastStorageError: _lastStorageError,
|
||||
LastReadySucceeded: _lastReadySucceeded);
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkBootstrapCompleted(TimeSpan duration)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
_bootstrapCompletedAt = completedAt;
|
||||
_bootstrapDuration = duration;
|
||||
_lastReadySucceeded = true;
|
||||
_lastStorageLatency = duration;
|
||||
_lastStorageError = null;
|
||||
_lastReadyCheckAt = completedAt;
|
||||
}
|
||||
}
|
||||
|
||||
public void RecordStorageCheck(bool success, TimeSpan latency, string? error)
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
_lastReadySucceeded = success;
|
||||
_lastStorageLatency = latency;
|
||||
_lastStorageError = success ? null : error;
|
||||
_lastReadyCheckAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record ServiceHealthSnapshot(
|
||||
DateTimeOffset CapturedAt,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset? BootstrapCompletedAt,
|
||||
TimeSpan? BootstrapDuration,
|
||||
DateTimeOffset? LastReadyCheckAt,
|
||||
TimeSpan? LastStorageLatency,
|
||||
string? LastStorageError,
|
||||
bool LastReadySucceeded);
|
||||
|
||||
@@ -5,16 +5,16 @@ using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Concelier.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
internal static class AdvisoryRawRequestMapper
|
||||
{
|
||||
internal static AdvisoryRawDocument Map(AdvisoryIngestRequest request, string tenant, TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
var sourceRequest = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
|
||||
|
||||
internal static class AdvisoryRawRequestMapper
|
||||
{
|
||||
internal static AdvisoryRawDocument Map(AdvisoryIngestRequest request, string tenant, TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
var sourceRequest = request.Source ?? throw new ArgumentException("source section is required.", nameof(request));
|
||||
var upstreamRequest = request.Upstream ?? throw new ArgumentException("upstream section is required.", nameof(request));
|
||||
var contentRequest = request.Content ?? throw new ArgumentException("content section is required.", nameof(request));
|
||||
var identifiersRequest = request.Identifiers ?? throw new ArgumentException("identifiers section is required.", nameof(request));
|
||||
@@ -22,18 +22,18 @@ internal static class AdvisoryRawRequestMapper
|
||||
var source = new RawSourceMetadata(
|
||||
sourceRequest.Vendor,
|
||||
sourceRequest.Connector,
|
||||
sourceRequest.Version,
|
||||
string.IsNullOrWhiteSpace(sourceRequest.Stream) ? null : sourceRequest.Stream);
|
||||
|
||||
var signatureRequest = upstreamRequest.Signature ?? new AdvisorySignatureRequest(false, null, null, null, null, null);
|
||||
var signature = new RawSignatureMetadata(
|
||||
signatureRequest.Present,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Format) ? null : signatureRequest.Format,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.KeyId) ? null : signatureRequest.KeyId,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Signature) ? null : signatureRequest.Signature,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Certificate) ? null : signatureRequest.Certificate,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Digest) ? null : signatureRequest.Digest);
|
||||
|
||||
sourceRequest.Version,
|
||||
string.IsNullOrWhiteSpace(sourceRequest.Stream) ? null : sourceRequest.Stream);
|
||||
|
||||
var signatureRequest = upstreamRequest.Signature ?? new AdvisorySignatureRequest(false, null, null, null, null, null);
|
||||
var signature = new RawSignatureMetadata(
|
||||
signatureRequest.Present,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Format) ? null : signatureRequest.Format,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.KeyId) ? null : signatureRequest.KeyId,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Signature) ? null : signatureRequest.Signature,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Certificate) ? null : signatureRequest.Certificate,
|
||||
string.IsNullOrWhiteSpace(signatureRequest.Digest) ? null : signatureRequest.Digest);
|
||||
|
||||
var retrievedAt = upstreamRequest.RetrievedAt ?? timeProvider.GetUtcNow();
|
||||
var upstream = new RawUpstreamMetadata(
|
||||
upstreamRequest.UpstreamId,
|
||||
@@ -49,13 +49,13 @@ internal static class AdvisoryRawRequestMapper
|
||||
string.IsNullOrWhiteSpace(contentRequest.SpecVersion) ? null : contentRequest.SpecVersion,
|
||||
rawContent,
|
||||
string.IsNullOrWhiteSpace(contentRequest.Encoding) ? null : contentRequest.Encoding);
|
||||
|
||||
var aliases = NormalizeStrings(identifiersRequest.Aliases);
|
||||
if (aliases.IsDefault)
|
||||
{
|
||||
aliases = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
|
||||
var aliases = NormalizeStrings(identifiersRequest.Aliases);
|
||||
if (aliases.IsDefault)
|
||||
{
|
||||
aliases = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var identifiers = new RawIdentifiers(
|
||||
aliases,
|
||||
identifiersRequest.Primary);
|
||||
@@ -89,70 +89,70 @@ internal static class AdvisoryRawRequestMapper
|
||||
AdvisoryKey: advisoryKey,
|
||||
Links: links);
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<string>();
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
internal static ImmutableDictionary<string, string> NormalizeDictionary(IDictionary<string, string>? values)
|
||||
{
|
||||
if (values is null || values.Count == 0)
|
||||
{
|
||||
return ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var kv in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kv.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder[kv.Key.Trim()] = kv.Value?.Trim() ?? string.Empty;
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
|
||||
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<string>();
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(value.Trim());
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
internal static ImmutableDictionary<string, string> NormalizeDictionary(IDictionary<string, string>? values)
|
||||
{
|
||||
if (values is null || values.Count == 0)
|
||||
{
|
||||
return ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var kv in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kv.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder[kv.Key.Trim()] = kv.Value?.Trim() ?? string.Empty;
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<RawReference> NormalizeReferences(IEnumerable<AdvisoryLinksetReferenceRequest>? references)
|
||||
{
|
||||
if (references is null)
|
||||
{
|
||||
return ImmutableArray<RawReference>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RawReference>();
|
||||
foreach (var reference in references)
|
||||
{
|
||||
if (reference is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(reference.Type) || string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(new RawReference(reference.Type.Trim(), reference.Url.Trim(), string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RawReference>();
|
||||
foreach (var reference in references)
|
||||
{
|
||||
if (reference is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(reference.Type) || string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(new RawReference(reference.Type.Trim(), reference.Url.Trim(), string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<RawReference>.Empty : builder.ToImmutable();
|
||||
@@ -185,7 +185,7 @@ internal static class AdvisoryRawRequestMapper
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<RawRelationship>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
|
||||
private static JsonElement NormalizeRawContent(JsonElement element)
|
||||
{
|
||||
var json = element.ValueKind == JsonValueKind.Undefined ? "{}" : element.GetRawText();
|
||||
|
||||
@@ -1,38 +1,38 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using YamlDotNet.Serialization;
|
||||
using YamlDotNet.Serialization.NamingConventions;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
public static class ConfigurationExtensions
|
||||
{
|
||||
public static IConfigurationBuilder AddConcelierYaml(this IConfigurationBuilder builder, string path)
|
||||
{
|
||||
if (builder is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(builder));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
|
||||
{
|
||||
return builder;
|
||||
}
|
||||
|
||||
var deserializer = new DeserializerBuilder()
|
||||
.WithNamingConvention(CamelCaseNamingConvention.Instance)
|
||||
.Build();
|
||||
|
||||
using var reader = File.OpenText(path);
|
||||
var yamlObject = deserializer.Deserialize(reader);
|
||||
if (yamlObject is null)
|
||||
{
|
||||
return builder;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(yamlObject);
|
||||
var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
|
||||
return builder.AddJsonStream(stream);
|
||||
}
|
||||
}
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using YamlDotNet.Serialization;
|
||||
using YamlDotNet.Serialization.NamingConventions;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
public static class ConfigurationExtensions
|
||||
{
|
||||
public static IConfigurationBuilder AddConcelierYaml(this IConfigurationBuilder builder, string path)
|
||||
{
|
||||
if (builder is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(builder));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
|
||||
{
|
||||
return builder;
|
||||
}
|
||||
|
||||
var deserializer = new DeserializerBuilder()
|
||||
.WithNamingConvention(CamelCaseNamingConvention.Instance)
|
||||
.Build();
|
||||
|
||||
using var reader = File.OpenText(path);
|
||||
var yamlObject = deserializer.Deserialize(reader);
|
||||
if (yamlObject is null)
|
||||
{
|
||||
return builder;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(yamlObject);
|
||||
var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
|
||||
return builder.AddJsonStream(stream);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,208 +1,208 @@
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.WebService.Diagnostics;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
using StellaOps.Concelier.WebService.Services;
|
||||
using StellaOps.Concelier.WebService.Results;
|
||||
using HttpResults = Microsoft.AspNetCore.Http.Results;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
internal static class MirrorEndpointExtensions
|
||||
{
|
||||
private const string IndexScope = "index";
|
||||
private const string DownloadScope = "download";
|
||||
|
||||
public static void MapConcelierMirrorEndpoints(this WebApplication app, bool authorityConfigured, bool enforceAuthority)
|
||||
{
|
||||
app.MapGet("/concelier/exports/index.json", async (
|
||||
MirrorFileLocator locator,
|
||||
MirrorRateLimiter limiter,
|
||||
IOptionsMonitor<ConcelierOptions> optionsMonitor,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirrorOptions.Enabled)
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
|
||||
{
|
||||
return unauthorizedResult;
|
||||
}
|
||||
|
||||
if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter))
|
||||
{
|
||||
ApplyRetryAfter(context.Response, retryAfter);
|
||||
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
|
||||
}
|
||||
|
||||
if (!locator.TryResolveIndex(out var path, out _))
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
return await WriteFileAsync(context, path, "application/json").ConfigureAwait(false);
|
||||
});
|
||||
|
||||
app.MapGet("/concelier/exports/{**relativePath}", async (
|
||||
string? relativePath,
|
||||
MirrorFileLocator locator,
|
||||
MirrorRateLimiter limiter,
|
||||
IOptionsMonitor<ConcelierOptions> optionsMonitor,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirrorOptions.Enabled)
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId))
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context, relativePath);
|
||||
}
|
||||
|
||||
var domain = FindDomain(mirrorOptions, domainId);
|
||||
|
||||
if (!TryAuthorize(domain?.RequireAuthentication ?? mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
|
||||
{
|
||||
return unauthorizedResult;
|
||||
}
|
||||
|
||||
var limit = domain?.MaxDownloadRequestsPerHour ?? mirrorOptions.MaxIndexRequestsPerHour;
|
||||
if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter))
|
||||
{
|
||||
ApplyRetryAfter(context.Response, retryAfter);
|
||||
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
|
||||
}
|
||||
|
||||
var contentType = ResolveContentType(path);
|
||||
return await WriteFileAsync(context, path, contentType).ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
|
||||
private static ConcelierOptions.MirrorDomainOptions? FindDomain(ConcelierOptions.MirrorOptions mirrorOptions, string? domainId)
|
||||
{
|
||||
if (domainId is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var candidate in mirrorOptions.Domains)
|
||||
{
|
||||
if (candidate is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(candidate.Id, domainId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
|
||||
{
|
||||
result = HttpResults.Empty;
|
||||
if (!requireAuthentication)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!enforceAuthority || !authorityConfigured)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (context.User?.Identity?.IsAuthenticated == true)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
|
||||
result = HttpResults.StatusCode(StatusCodes.Status401Unauthorized);
|
||||
return false;
|
||||
}
|
||||
|
||||
private static Task<IResult> WriteFileAsync(HttpContext context, string path, string contentType)
|
||||
{
|
||||
var fileInfo = new FileInfo(path);
|
||||
if (!fileInfo.Exists)
|
||||
{
|
||||
return Task.FromResult(ConcelierProblemResultFactory.MirrorNotFound(context, path));
|
||||
}
|
||||
|
||||
var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read | FileShare.Delete);
|
||||
|
||||
context.Response.Headers.CacheControl = BuildCacheControlHeader(path);
|
||||
context.Response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
|
||||
context.Response.ContentLength = fileInfo.Length;
|
||||
return Task.FromResult(HttpResults.Stream(stream, contentType));
|
||||
}
|
||||
|
||||
private static string ResolveContentType(string path)
|
||||
{
|
||||
if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "application/json";
|
||||
}
|
||||
|
||||
if (path.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "application/jose+json";
|
||||
}
|
||||
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
private static void ApplyRetryAfter(HttpResponse response, TimeSpan? retryAfter)
|
||||
{
|
||||
if (retryAfter is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1);
|
||||
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static string BuildCacheControlHeader(string path)
|
||||
{
|
||||
var fileName = Path.GetFileName(path);
|
||||
if (fileName is null)
|
||||
{
|
||||
return "public, max-age=60";
|
||||
}
|
||||
|
||||
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "public, max-age=60";
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
|
||||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "public, max-age=300, immutable";
|
||||
}
|
||||
|
||||
return "public, max-age=300";
|
||||
}
|
||||
}
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.WebService.Diagnostics;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
using StellaOps.Concelier.WebService.Services;
|
||||
using StellaOps.Concelier.WebService.Results;
|
||||
using HttpResults = Microsoft.AspNetCore.Http.Results;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Extensions;
|
||||
|
||||
internal static class MirrorEndpointExtensions
|
||||
{
|
||||
private const string IndexScope = "index";
|
||||
private const string DownloadScope = "download";
|
||||
|
||||
public static void MapConcelierMirrorEndpoints(this WebApplication app, bool authorityConfigured, bool enforceAuthority)
|
||||
{
|
||||
app.MapGet("/concelier/exports/index.json", async (
|
||||
MirrorFileLocator locator,
|
||||
MirrorRateLimiter limiter,
|
||||
IOptionsMonitor<ConcelierOptions> optionsMonitor,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirrorOptions.Enabled)
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
|
||||
{
|
||||
return unauthorizedResult;
|
||||
}
|
||||
|
||||
if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter))
|
||||
{
|
||||
ApplyRetryAfter(context.Response, retryAfter);
|
||||
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
|
||||
}
|
||||
|
||||
if (!locator.TryResolveIndex(out var path, out _))
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
return await WriteFileAsync(context, path, "application/json").ConfigureAwait(false);
|
||||
});
|
||||
|
||||
app.MapGet("/concelier/exports/{**relativePath}", async (
|
||||
string? relativePath,
|
||||
MirrorFileLocator locator,
|
||||
MirrorRateLimiter limiter,
|
||||
IOptionsMonitor<ConcelierOptions> optionsMonitor,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirrorOptions.Enabled)
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context);
|
||||
}
|
||||
|
||||
if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId))
|
||||
{
|
||||
return ConcelierProblemResultFactory.MirrorNotFound(context, relativePath);
|
||||
}
|
||||
|
||||
var domain = FindDomain(mirrorOptions, domainId);
|
||||
|
||||
if (!TryAuthorize(domain?.RequireAuthentication ?? mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
|
||||
{
|
||||
return unauthorizedResult;
|
||||
}
|
||||
|
||||
var limit = domain?.MaxDownloadRequestsPerHour ?? mirrorOptions.MaxIndexRequestsPerHour;
|
||||
if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter))
|
||||
{
|
||||
ApplyRetryAfter(context.Response, retryAfter);
|
||||
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
|
||||
}
|
||||
|
||||
var contentType = ResolveContentType(path);
|
||||
return await WriteFileAsync(context, path, contentType).ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
|
||||
private static ConcelierOptions.MirrorDomainOptions? FindDomain(ConcelierOptions.MirrorOptions mirrorOptions, string? domainId)
|
||||
{
|
||||
if (domainId is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var candidate in mirrorOptions.Domains)
|
||||
{
|
||||
if (candidate is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(candidate.Id, domainId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
|
||||
{
|
||||
result = HttpResults.Empty;
|
||||
if (!requireAuthentication)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!enforceAuthority || !authorityConfigured)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (context.User?.Identity?.IsAuthenticated == true)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
|
||||
result = HttpResults.StatusCode(StatusCodes.Status401Unauthorized);
|
||||
return false;
|
||||
}
|
||||
|
||||
private static Task<IResult> WriteFileAsync(HttpContext context, string path, string contentType)
|
||||
{
|
||||
var fileInfo = new FileInfo(path);
|
||||
if (!fileInfo.Exists)
|
||||
{
|
||||
return Task.FromResult(ConcelierProblemResultFactory.MirrorNotFound(context, path));
|
||||
}
|
||||
|
||||
var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read | FileShare.Delete);
|
||||
|
||||
context.Response.Headers.CacheControl = BuildCacheControlHeader(path);
|
||||
context.Response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
|
||||
context.Response.ContentLength = fileInfo.Length;
|
||||
return Task.FromResult(HttpResults.Stream(stream, contentType));
|
||||
}
|
||||
|
||||
private static string ResolveContentType(string path)
|
||||
{
|
||||
if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "application/json";
|
||||
}
|
||||
|
||||
if (path.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "application/jose+json";
|
||||
}
|
||||
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
private static void ApplyRetryAfter(HttpResponse response, TimeSpan? retryAfter)
|
||||
{
|
||||
if (retryAfter is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1);
|
||||
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static string BuildCacheControlHeader(string path)
|
||||
{
|
||||
var fileName = Path.GetFileName(path);
|
||||
if (fileName is null)
|
||||
{
|
||||
return "public, max-age=60";
|
||||
}
|
||||
|
||||
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "public, max-age=60";
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
|
||||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "public, max-age=300, immutable";
|
||||
}
|
||||
|
||||
return "public, max-age=300";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,104 +1,104 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Filters;
|
||||
|
||||
/// <summary>
|
||||
/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage.
|
||||
/// </summary>
|
||||
public sealed class JobAuthorizationAuditFilter : IEndpointFilter
|
||||
{
|
||||
internal const string LoggerName = "Concelier.Authorization.Audit";
|
||||
|
||||
public async ValueTask<object?> InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(next);
|
||||
|
||||
var httpContext = context.HttpContext;
|
||||
var options = httpContext.RequestServices.GetRequiredService<IOptions<ConcelierOptions>>().Value;
|
||||
var authority = options.Authority;
|
||||
|
||||
if (authority is null || !authority.Enabled)
|
||||
{
|
||||
return await next(context).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var logger = httpContext.RequestServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger(LoggerName);
|
||||
|
||||
var remoteAddress = httpContext.Connection.RemoteIpAddress;
|
||||
var matcher = new NetworkMaskMatcher(authority.BypassNetworks);
|
||||
var user = httpContext.User;
|
||||
var isAuthenticated = user?.Identity?.IsAuthenticated ?? false;
|
||||
var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress);
|
||||
|
||||
var result = await next(context).ConfigureAwait(false);
|
||||
|
||||
var scopes = ExtractScopes(user);
|
||||
var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value;
|
||||
var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value;
|
||||
|
||||
logger.LogInformation(
|
||||
"Concelier authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}",
|
||||
httpContext.Request.Path.Value ?? string.Empty,
|
||||
httpContext.Response.StatusCode,
|
||||
string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject,
|
||||
string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId,
|
||||
scopes.Length == 0 ? "(none)" : string.Join(',', scopes),
|
||||
bypassUsed,
|
||||
remoteAddress?.ToString() ?? IPAddress.None.ToString());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string[] ExtractScopes(ClaimsPrincipal? principal)
|
||||
{
|
||||
if (principal is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var values = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
values.Add(claim.Value);
|
||||
}
|
||||
|
||||
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var normalized = StellaOpsScopes.Normalize(part);
|
||||
if (!string.IsNullOrEmpty(normalized))
|
||||
{
|
||||
values.Add(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return values.Count == 0 ? Array.Empty<string>() : values.ToArray();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Filters;
|
||||
|
||||
/// <summary>
|
||||
/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage.
|
||||
/// </summary>
|
||||
public sealed class JobAuthorizationAuditFilter : IEndpointFilter
|
||||
{
|
||||
internal const string LoggerName = "Concelier.Authorization.Audit";
|
||||
|
||||
public async ValueTask<object?> InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(next);
|
||||
|
||||
var httpContext = context.HttpContext;
|
||||
var options = httpContext.RequestServices.GetRequiredService<IOptions<ConcelierOptions>>().Value;
|
||||
var authority = options.Authority;
|
||||
|
||||
if (authority is null || !authority.Enabled)
|
||||
{
|
||||
return await next(context).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var logger = httpContext.RequestServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger(LoggerName);
|
||||
|
||||
var remoteAddress = httpContext.Connection.RemoteIpAddress;
|
||||
var matcher = new NetworkMaskMatcher(authority.BypassNetworks);
|
||||
var user = httpContext.User;
|
||||
var isAuthenticated = user?.Identity?.IsAuthenticated ?? false;
|
||||
var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress);
|
||||
|
||||
var result = await next(context).ConfigureAwait(false);
|
||||
|
||||
var scopes = ExtractScopes(user);
|
||||
var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value;
|
||||
var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value;
|
||||
|
||||
logger.LogInformation(
|
||||
"Concelier authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}",
|
||||
httpContext.Request.Path.Value ?? string.Empty,
|
||||
httpContext.Response.StatusCode,
|
||||
string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject,
|
||||
string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId,
|
||||
scopes.Length == 0 ? "(none)" : string.Join(',', scopes),
|
||||
bypassUsed,
|
||||
remoteAddress?.ToString() ?? IPAddress.None.ToString());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string[] ExtractScopes(ClaimsPrincipal? principal)
|
||||
{
|
||||
if (principal is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var values = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
values.Add(claim.Value);
|
||||
}
|
||||
|
||||
foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(claim.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var normalized = StellaOpsScopes.Normalize(part);
|
||||
if (!string.IsNullOrEmpty(normalized))
|
||||
{
|
||||
values.Add(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return values.Count == 0 ? Array.Empty<string>() : values.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed record JobDefinitionResponse(
|
||||
string Kind,
|
||||
bool Enabled,
|
||||
string? CronExpression,
|
||||
TimeSpan Timeout,
|
||||
TimeSpan LeaseDuration,
|
||||
JobRunResponse? LastRun)
|
||||
{
|
||||
public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun)
|
||||
{
|
||||
return new JobDefinitionResponse(
|
||||
definition.Kind,
|
||||
definition.Enabled,
|
||||
definition.CronExpression,
|
||||
definition.Timeout,
|
||||
definition.LeaseDuration,
|
||||
lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun));
|
||||
}
|
||||
}
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed record JobDefinitionResponse(
|
||||
string Kind,
|
||||
bool Enabled,
|
||||
string? CronExpression,
|
||||
TimeSpan Timeout,
|
||||
TimeSpan LeaseDuration,
|
||||
JobRunResponse? LastRun)
|
||||
{
|
||||
public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun)
|
||||
{
|
||||
return new JobDefinitionResponse(
|
||||
definition.Kind,
|
||||
definition.Enabled,
|
||||
definition.CronExpression,
|
||||
definition.Timeout,
|
||||
definition.LeaseDuration,
|
||||
lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +1,29 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed record JobRunResponse(
|
||||
Guid RunId,
|
||||
string Kind,
|
||||
JobRunStatus Status,
|
||||
string Trigger,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset? StartedAt,
|
||||
DateTimeOffset? CompletedAt,
|
||||
string? Error,
|
||||
TimeSpan? Duration,
|
||||
IReadOnlyDictionary<string, object?> Parameters)
|
||||
{
|
||||
public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot)
|
||||
=> new(
|
||||
snapshot.RunId,
|
||||
snapshot.Kind,
|
||||
snapshot.Status,
|
||||
snapshot.Trigger,
|
||||
snapshot.CreatedAt,
|
||||
snapshot.StartedAt,
|
||||
snapshot.CompletedAt,
|
||||
snapshot.Error,
|
||||
snapshot.Duration,
|
||||
snapshot.Parameters);
|
||||
}
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed record JobRunResponse(
|
||||
Guid RunId,
|
||||
string Kind,
|
||||
JobRunStatus Status,
|
||||
string Trigger,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset? StartedAt,
|
||||
DateTimeOffset? CompletedAt,
|
||||
string? Error,
|
||||
TimeSpan? Duration,
|
||||
IReadOnlyDictionary<string, object?> Parameters)
|
||||
{
|
||||
public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot)
|
||||
=> new(
|
||||
snapshot.RunId,
|
||||
snapshot.Kind,
|
||||
snapshot.Status,
|
||||
snapshot.Trigger,
|
||||
snapshot.CreatedAt,
|
||||
snapshot.StartedAt,
|
||||
snapshot.CompletedAt,
|
||||
snapshot.Error,
|
||||
snapshot.Duration,
|
||||
snapshot.Parameters);
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed class JobTriggerRequest
|
||||
{
|
||||
public string Trigger { get; set; } = "api";
|
||||
|
||||
public Dictionary<string, object?> Parameters { get; set; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
namespace StellaOps.Concelier.WebService.Jobs;
|
||||
|
||||
public sealed class JobTriggerRequest
|
||||
{
|
||||
public string Trigger { get; set; } = "api";
|
||||
|
||||
public Dictionary<string, object?> Parameters { get; set; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
@@ -1,71 +1,71 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Post-configuration helpers for <see cref="ConcelierOptions"/>.
|
||||
/// </summary>
|
||||
public static class ConcelierOptionsPostConfigure
|
||||
{
|
||||
/// <summary>
|
||||
/// Applies derived settings that require filesystem access, such as loading client secrets from disk.
|
||||
/// </summary>
|
||||
/// <param name="options">The options to mutate.</param>
|
||||
/// <param name="contentRootPath">Application content root used to resolve relative paths.</param>
|
||||
public static void Apply(ConcelierOptions options, string contentRootPath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Post-configuration helpers for <see cref="ConcelierOptions"/>.
|
||||
/// </summary>
|
||||
public static class ConcelierOptionsPostConfigure
|
||||
{
|
||||
/// <summary>
|
||||
/// Applies derived settings that require filesystem access, such as loading client secrets from disk.
|
||||
/// </summary>
|
||||
/// <param name="options">The options to mutate.</param>
|
||||
/// <param name="contentRootPath">Application content root used to resolve relative paths.</param>
|
||||
public static void Apply(ConcelierOptions options, string contentRootPath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
options.Authority ??= new ConcelierOptions.AuthorityOptions();
|
||||
options.Features ??= new ConcelierOptions.FeaturesOptions();
|
||||
options.Evidence ??= new ConcelierOptions.EvidenceBundleOptions();
|
||||
|
||||
var authority = options.Authority;
|
||||
if (string.IsNullOrWhiteSpace(authority.ClientSecret)
|
||||
&& !string.IsNullOrWhiteSpace(authority.ClientSecretFile))
|
||||
{
|
||||
var resolvedPath = authority.ClientSecretFile!;
|
||||
if (!Path.IsPathRooted(resolvedPath))
|
||||
{
|
||||
resolvedPath = Path.Combine(contentRootPath, resolvedPath);
|
||||
}
|
||||
|
||||
if (!File.Exists(resolvedPath))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found.");
|
||||
}
|
||||
|
||||
var secret = File.ReadAllText(resolvedPath).Trim();
|
||||
if (string.IsNullOrEmpty(secret))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty.");
|
||||
}
|
||||
|
||||
authority.ClientSecret = secret;
|
||||
}
|
||||
|
||||
|
||||
var authority = options.Authority;
|
||||
if (string.IsNullOrWhiteSpace(authority.ClientSecret)
|
||||
&& !string.IsNullOrWhiteSpace(authority.ClientSecretFile))
|
||||
{
|
||||
var resolvedPath = authority.ClientSecretFile!;
|
||||
if (!Path.IsPathRooted(resolvedPath))
|
||||
{
|
||||
resolvedPath = Path.Combine(contentRootPath, resolvedPath);
|
||||
}
|
||||
|
||||
if (!File.Exists(resolvedPath))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found.");
|
||||
}
|
||||
|
||||
var secret = File.ReadAllText(resolvedPath).Trim();
|
||||
if (string.IsNullOrEmpty(secret))
|
||||
{
|
||||
throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty.");
|
||||
}
|
||||
|
||||
authority.ClientSecret = secret;
|
||||
}
|
||||
|
||||
options.Mirror ??= new ConcelierOptions.MirrorOptions();
|
||||
var mirror = options.Mirror;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
|
||||
{
|
||||
mirror.ExportRoot = Path.Combine("exports", "json");
|
||||
}
|
||||
|
||||
var resolvedRoot = mirror.ExportRoot;
|
||||
if (!Path.IsPathRooted(resolvedRoot))
|
||||
{
|
||||
resolvedRoot = Path.Combine(contentRootPath, resolvedRoot);
|
||||
}
|
||||
|
||||
mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName))
|
||||
{
|
||||
mirror.LatestDirectoryName = "latest";
|
||||
}
|
||||
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
|
||||
{
|
||||
mirror.ExportRoot = Path.Combine("exports", "json");
|
||||
}
|
||||
|
||||
var resolvedRoot = mirror.ExportRoot;
|
||||
if (!Path.IsPathRooted(resolvedRoot))
|
||||
{
|
||||
resolvedRoot = Path.Combine(contentRootPath, resolvedRoot);
|
||||
}
|
||||
|
||||
mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName))
|
||||
{
|
||||
mirror.LatestDirectoryName = "latest";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName))
|
||||
{
|
||||
mirror.MirrorDirectoryName = "mirror";
|
||||
|
||||
@@ -65,7 +65,7 @@ using StellaOps.Aoc.AspNetCore.Results;
|
||||
using HttpResults = Microsoft.AspNetCore.Http.Results;
|
||||
using StellaOps.Concelier.Storage.Advisories;
|
||||
using StellaOps.Concelier.Storage.Aliases;
|
||||
using StellaOps.Provenance.Mongo;
|
||||
using StellaOps.Provenance;
|
||||
|
||||
namespace StellaOps.Concelier.WebService
|
||||
{
|
||||
|
||||
@@ -1,184 +1,184 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
internal sealed class MirrorFileLocator
|
||||
{
|
||||
private readonly IOptionsMonitor<ConcelierOptions> _options;
|
||||
private readonly ILogger<MirrorFileLocator> _logger;
|
||||
|
||||
public MirrorFileLocator(IOptionsMonitor<ConcelierOptions> options, ILogger<MirrorFileLocator> logger)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public bool TryResolveIndex([NotNullWhen(true)] out string? path, [NotNullWhen(true)] out string? exportId)
|
||||
=> TryResolveRelativePath("index.json", out path, out exportId, out _);
|
||||
|
||||
public bool TryResolveRelativePath(string relativePath, [NotNullWhen(true)] out string? fullPath, [NotNullWhen(true)] out string? exportId, out string? domainId)
|
||||
{
|
||||
fullPath = null;
|
||||
exportId = null;
|
||||
domainId = null;
|
||||
|
||||
var mirror = _options.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirror.Enabled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryResolveExportDirectory(mirror, out var exportDirectory, out exportId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var sanitized = SanitizeRelativePath(relativePath);
|
||||
if (sanitized.Length == 0 || string.Equals(sanitized, "index.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
sanitized = $"{mirror.MirrorDirectoryName}/index.json";
|
||||
}
|
||||
|
||||
if (!sanitized.StartsWith($"{mirror.MirrorDirectoryName}/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var candidate = Combine(exportDirectory, sanitized);
|
||||
if (!CandidateWithinExport(exportDirectory, candidate))
|
||||
{
|
||||
_logger.LogWarning("Rejected mirror export request for path '{RelativePath}' due to traversal attempt.", relativePath);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!File.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Extract domain id from path mirror/<domain>/...
|
||||
var segments = sanitized.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length >= 2)
|
||||
{
|
||||
domainId = segments[1];
|
||||
}
|
||||
|
||||
fullPath = candidate;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryResolveExportDirectory(ConcelierOptions.MirrorOptions mirror, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
|
||||
{
|
||||
exportDirectory = null;
|
||||
exportId = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute))
|
||||
{
|
||||
_logger.LogWarning("Mirror export root is not configured; unable to serve mirror content.");
|
||||
return false;
|
||||
}
|
||||
|
||||
var root = mirror.ExportRootAbsolute;
|
||||
var candidateSegment = string.IsNullOrWhiteSpace(mirror.ActiveExportId)
|
||||
? mirror.LatestDirectoryName
|
||||
: mirror.ActiveExportId!;
|
||||
|
||||
if (TryResolveCandidate(root, candidateSegment, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.Equals(candidateSegment, mirror.LatestDirectoryName, StringComparison.OrdinalIgnoreCase)
|
||||
&& TryResolveCandidate(root, mirror.LatestDirectoryName, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var directories = Directory.Exists(root)
|
||||
? Directory.GetDirectories(root)
|
||||
: Array.Empty<string>();
|
||||
|
||||
Array.Sort(directories, StringComparer.Ordinal);
|
||||
Array.Reverse(directories);
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
if (TryResolveCandidate(root, Path.GetFileName(directory), mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to enumerate export directories under {Root}.", root);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool TryResolveCandidate(string root, string segment, string mirrorDirectory, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
|
||||
{
|
||||
exportDirectory = null;
|
||||
exportId = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var candidate = Path.Combine(root, segment);
|
||||
if (!Directory.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var mirrorPath = Path.Combine(candidate, mirrorDirectory);
|
||||
if (!Directory.Exists(mirrorPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
exportDirectory = candidate;
|
||||
exportId = segment;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string SanitizeRelativePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = relativePath.Replace('\\', '/').Trim().TrimStart('/');
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private static string Combine(string root, string relativePath)
|
||||
{
|
||||
var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
return Path.GetFullPath(root);
|
||||
}
|
||||
|
||||
var combinedRelative = Path.Combine(segments);
|
||||
return Path.GetFullPath(Path.Combine(root, combinedRelative));
|
||||
}
|
||||
|
||||
private static bool CandidateWithinExport(string exportDirectory, string candidate)
|
||||
{
|
||||
var exportRoot = Path.GetFullPath(exportDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
var candidatePath = Path.GetFullPath(candidate);
|
||||
return candidatePath.StartsWith(exportRoot, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.WebService.Options;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
internal sealed class MirrorFileLocator
|
||||
{
|
||||
private readonly IOptionsMonitor<ConcelierOptions> _options;
|
||||
private readonly ILogger<MirrorFileLocator> _logger;
|
||||
|
||||
public MirrorFileLocator(IOptionsMonitor<ConcelierOptions> options, ILogger<MirrorFileLocator> logger)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public bool TryResolveIndex([NotNullWhen(true)] out string? path, [NotNullWhen(true)] out string? exportId)
|
||||
=> TryResolveRelativePath("index.json", out path, out exportId, out _);
|
||||
|
||||
public bool TryResolveRelativePath(string relativePath, [NotNullWhen(true)] out string? fullPath, [NotNullWhen(true)] out string? exportId, out string? domainId)
|
||||
{
|
||||
fullPath = null;
|
||||
exportId = null;
|
||||
domainId = null;
|
||||
|
||||
var mirror = _options.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
|
||||
if (!mirror.Enabled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryResolveExportDirectory(mirror, out var exportDirectory, out exportId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var sanitized = SanitizeRelativePath(relativePath);
|
||||
if (sanitized.Length == 0 || string.Equals(sanitized, "index.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
sanitized = $"{mirror.MirrorDirectoryName}/index.json";
|
||||
}
|
||||
|
||||
if (!sanitized.StartsWith($"{mirror.MirrorDirectoryName}/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var candidate = Combine(exportDirectory, sanitized);
|
||||
if (!CandidateWithinExport(exportDirectory, candidate))
|
||||
{
|
||||
_logger.LogWarning("Rejected mirror export request for path '{RelativePath}' due to traversal attempt.", relativePath);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!File.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Extract domain id from path mirror/<domain>/...
|
||||
var segments = sanitized.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length >= 2)
|
||||
{
|
||||
domainId = segments[1];
|
||||
}
|
||||
|
||||
fullPath = candidate;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryResolveExportDirectory(ConcelierOptions.MirrorOptions mirror, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
|
||||
{
|
||||
exportDirectory = null;
|
||||
exportId = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute))
|
||||
{
|
||||
_logger.LogWarning("Mirror export root is not configured; unable to serve mirror content.");
|
||||
return false;
|
||||
}
|
||||
|
||||
var root = mirror.ExportRootAbsolute;
|
||||
var candidateSegment = string.IsNullOrWhiteSpace(mirror.ActiveExportId)
|
||||
? mirror.LatestDirectoryName
|
||||
: mirror.ActiveExportId!;
|
||||
|
||||
if (TryResolveCandidate(root, candidateSegment, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.Equals(candidateSegment, mirror.LatestDirectoryName, StringComparison.OrdinalIgnoreCase)
|
||||
&& TryResolveCandidate(root, mirror.LatestDirectoryName, mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var directories = Directory.Exists(root)
|
||||
? Directory.GetDirectories(root)
|
||||
: Array.Empty<string>();
|
||||
|
||||
Array.Sort(directories, StringComparer.Ordinal);
|
||||
Array.Reverse(directories);
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
if (TryResolveCandidate(root, Path.GetFileName(directory), mirror.MirrorDirectoryName, out exportDirectory, out exportId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to enumerate export directories under {Root}.", root);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool TryResolveCandidate(string root, string segment, string mirrorDirectory, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId)
|
||||
{
|
||||
exportDirectory = null;
|
||||
exportId = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var candidate = Path.Combine(root, segment);
|
||||
if (!Directory.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var mirrorPath = Path.Combine(candidate, mirrorDirectory);
|
||||
if (!Directory.Exists(mirrorPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
exportDirectory = candidate;
|
||||
exportId = segment;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string SanitizeRelativePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = relativePath.Replace('\\', '/').Trim().TrimStart('/');
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private static string Combine(string root, string relativePath)
|
||||
{
|
||||
var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
return Path.GetFullPath(root);
|
||||
}
|
||||
|
||||
var combinedRelative = Path.Combine(segments);
|
||||
return Path.GetFullPath(Path.Combine(root, combinedRelative));
|
||||
}
|
||||
|
||||
private static bool CandidateWithinExport(string exportDirectory, string candidate)
|
||||
{
|
||||
var exportRoot = Path.GetFullPath(exportDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
var candidatePath = Path.GetFullPath(candidate);
|
||||
return candidatePath.StartsWith(exportRoot, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,57 +1,57 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
internal sealed class MirrorRateLimiter
|
||||
{
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
|
||||
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
|
||||
{
|
||||
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
|
||||
{
|
||||
retryAfter = null;
|
||||
|
||||
if (limit <= 0 || limit == int.MaxValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = CreateKey(domainId, scope);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var counter = _cache.Get<Counter>(key);
|
||||
if (counter is null || now - counter.WindowStart >= Window)
|
||||
{
|
||||
counter = new Counter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= limit)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + Window;
|
||||
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return false;
|
||||
}
|
||||
|
||||
counter = counter with { Count = counter.Count + 1 };
|
||||
var absoluteExpiration = counter.WindowStart + Window;
|
||||
_cache.Set(key, counter, absoluteExpiration);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateKey(string domainId, string scope)
|
||||
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
|
||||
{
|
||||
state.domainId.AsSpan().CopyTo(span);
|
||||
span[state.domainId.Length] = '|';
|
||||
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private sealed record Counter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace StellaOps.Concelier.WebService.Services;
|
||||
|
||||
internal sealed class MirrorRateLimiter
|
||||
{
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
|
||||
public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider)
|
||||
{
|
||||
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter)
|
||||
{
|
||||
retryAfter = null;
|
||||
|
||||
if (limit <= 0 || limit == int.MaxValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = CreateKey(domainId, scope);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var counter = _cache.Get<Counter>(key);
|
||||
if (counter is null || now - counter.WindowStart >= Window)
|
||||
{
|
||||
counter = new Counter(now, 0);
|
||||
}
|
||||
|
||||
if (counter.Count >= limit)
|
||||
{
|
||||
var windowEnd = counter.WindowStart + Window;
|
||||
retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero;
|
||||
return false;
|
||||
}
|
||||
|
||||
counter = counter with { Count = counter.Count + 1 };
|
||||
var absoluteExpiration = counter.WindowStart + Window;
|
||||
_cache.Set(key, counter, absoluteExpiration);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateKey(string domainId, string scope)
|
||||
=> string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) =>
|
||||
{
|
||||
state.domainId.AsSpan().CopyTo(span);
|
||||
span[state.domainId.Length] = '|';
|
||||
state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private sealed record Counter(DateTimeOffset WindowStart, int Count);
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@ using System.Xml.Linq;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Bson.IO;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Documents.IO;
|
||||
using StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
@@ -292,7 +292,7 @@ public sealed class AcscConnector : IFeedConnector
|
||||
var dto = AcscFeedParser.Parse(rawBytes, metadata.FeedSlug, parsedAt, _htmlSanitizer);
|
||||
|
||||
var json = JsonSerializer.Serialize(dto, SerializerOptions);
|
||||
var payload = BsonDocument.Parse(json);
|
||||
var payload = DocumentObject.Parse(json);
|
||||
|
||||
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
|
||||
var dtoRecord = existingDto is null
|
||||
@@ -678,7 +678,7 @@ public sealed class AcscConnector : IFeedConnector
|
||||
|
||||
private Task UpdateCursorAsync(AcscCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var document = cursor.ToDocumentObject();
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
public sealed class AcscConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "acsc";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
public sealed class AcscConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "acsc";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,44 +1,44 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:acsc";
|
||||
|
||||
private const string FetchCron = "7,37 * * * *";
|
||||
private const string ParseCron = "12,42 * * * *";
|
||||
private const string MapCron = "17,47 * * * *";
|
||||
private const string ProbeCron = "25,55 * * * *";
|
||||
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
|
||||
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
|
||||
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
|
||||
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddAcscConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler
|
||||
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
|
||||
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
|
||||
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
|
||||
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:acsc";
|
||||
|
||||
private const string FetchCron = "7,37 * * * *";
|
||||
private const string ParseCron = "12,42 * * * *";
|
||||
private const string MapCron = "17,47 * * * *";
|
||||
private const string ProbeCron = "25,55 * * * *";
|
||||
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
|
||||
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
|
||||
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
|
||||
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddAcscConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler
|
||||
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
|
||||
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
|
||||
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
|
||||
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,56 +1,56 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
public static class AcscServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<AcscOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = options.UserAgent;
|
||||
clientOptions.RequestVersion = options.RequestVersion;
|
||||
clientOptions.VersionPolicy = options.VersionPolicy;
|
||||
clientOptions.AllowAutoRedirect = true;
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
|
||||
handler.AllowAutoRedirect = true;
|
||||
};
|
||||
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
|
||||
if (options.RelayEndpoint is not null)
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
|
||||
}
|
||||
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
|
||||
{
|
||||
"application/rss+xml",
|
||||
"application/atom+xml;q=0.9",
|
||||
"application/xml;q=0.8",
|
||||
"text/xml;q=0.7",
|
||||
});
|
||||
});
|
||||
|
||||
services.AddSingleton<AcscDiagnostics>();
|
||||
services.AddTransient<AcscConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
public static class AcscServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<AcscOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = options.UserAgent;
|
||||
clientOptions.RequestVersion = options.RequestVersion;
|
||||
clientOptions.VersionPolicy = options.VersionPolicy;
|
||||
clientOptions.AllowAutoRedirect = true;
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
|
||||
handler.AllowAutoRedirect = true;
|
||||
};
|
||||
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
|
||||
if (options.RelayEndpoint is not null)
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
|
||||
}
|
||||
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
|
||||
{
|
||||
"application/rss+xml",
|
||||
"application/atom+xml;q=0.9",
|
||||
"application/xml;q=0.8",
|
||||
"text/xml;q=0.7",
|
||||
});
|
||||
});
|
||||
|
||||
services.AddSingleton<AcscDiagnostics>();
|
||||
services.AddTransient<AcscConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a single ACSC RSS feed endpoint.
|
||||
/// </summary>
|
||||
public sealed class AcscFeedOptions
|
||||
{
|
||||
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
/// <summary>
|
||||
/// Logical slug for the feed (alerts, advisories, threats, etc.).
|
||||
/// </summary>
|
||||
public string Slug { get; set; } = "alerts";
|
||||
|
||||
/// <summary>
|
||||
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
|
||||
/// </summary>
|
||||
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
|
||||
|
||||
/// <summary>
|
||||
/// Indicates whether the feed is active.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional display name for logging.
|
||||
/// </summary>
|
||||
public string? DisplayName { get; set; }
|
||||
|
||||
internal void Validate(int index)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Slug))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
|
||||
}
|
||||
|
||||
if (!SlugPattern.IsMatch(Slug))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(RelativePath))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
|
||||
}
|
||||
|
||||
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
|
||||
}
|
||||
}
|
||||
}
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a single ACSC RSS feed endpoint.
|
||||
/// </summary>
|
||||
public sealed class AcscFeedOptions
|
||||
{
|
||||
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
/// <summary>
|
||||
/// Logical slug for the feed (alerts, advisories, threats, etc.).
|
||||
/// </summary>
|
||||
public string Slug { get; set; } = "alerts";
|
||||
|
||||
/// <summary>
|
||||
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
|
||||
/// </summary>
|
||||
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
|
||||
|
||||
/// <summary>
|
||||
/// Indicates whether the feed is active.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional display name for logging.
|
||||
/// </summary>
|
||||
public string? DisplayName { get; set; }
|
||||
|
||||
internal void Validate(int index)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Slug))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
|
||||
}
|
||||
|
||||
if (!SlugPattern.IsMatch(Slug))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(RelativePath))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
|
||||
}
|
||||
|
||||
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,153 +1,153 @@
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options governing ACSC feed access and retry behaviour.
|
||||
/// </summary>
|
||||
public sealed class AcscOptions
|
||||
{
|
||||
public const string HttpClientName = "acsc";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
|
||||
|
||||
public AcscOptions()
|
||||
{
|
||||
Feeds = new List<AcscFeedOptions>
|
||||
{
|
||||
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
|
||||
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
|
||||
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
|
||||
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
|
||||
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint for direct ACSC fetches.
|
||||
/// </summary>
|
||||
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
|
||||
/// </summary>
|
||||
public Uri? RelayEndpoint { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
|
||||
/// </summary>
|
||||
public bool PreferRelayByDefault { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
|
||||
/// </summary>
|
||||
public bool EnableRelayFallback { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// If set, the connector will always use the relay endpoint and skip direct attempts.
|
||||
/// </summary>
|
||||
public bool ForceRelay { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to fetch requests (overrides HttpClient default).
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when marking fetch failures.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// Look-back period used when deriving initial published cursors.
|
||||
/// </summary>
|
||||
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
|
||||
|
||||
/// <summary>
|
||||
/// User-agent header sent with outbound requests.
|
||||
/// </summary>
|
||||
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
|
||||
|
||||
/// <summary>
|
||||
/// RSS feeds requested during fetch.
|
||||
/// </summary>
|
||||
public IList<AcscFeedOptions> Feeds { get; }
|
||||
|
||||
/// <summary>
|
||||
/// HTTP version policy requested for outbound requests.
|
||||
/// </summary>
|
||||
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
|
||||
|
||||
/// <summary>
|
||||
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
|
||||
/// </summary>
|
||||
public Version RequestVersion { get; set; } = HttpVersion.Version20;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
|
||||
}
|
||||
|
||||
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
|
||||
}
|
||||
|
||||
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (Feeds.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("At least one ACSC feed must be configured.");
|
||||
}
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
for (var i = 0; i < Feeds.Count; i++)
|
||||
{
|
||||
var feed = Feeds[i];
|
||||
feed.Validate(i);
|
||||
|
||||
if (!feed.Enabled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seen.Add(feed.Slug))
|
||||
{
|
||||
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options governing ACSC feed access and retry behaviour.
|
||||
/// </summary>
|
||||
public sealed class AcscOptions
|
||||
{
|
||||
public const string HttpClientName = "acsc";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
|
||||
|
||||
public AcscOptions()
|
||||
{
|
||||
Feeds = new List<AcscFeedOptions>
|
||||
{
|
||||
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
|
||||
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
|
||||
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
|
||||
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
|
||||
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint for direct ACSC fetches.
|
||||
/// </summary>
|
||||
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
|
||||
/// </summary>
|
||||
public Uri? RelayEndpoint { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
|
||||
/// </summary>
|
||||
public bool PreferRelayByDefault { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
|
||||
/// </summary>
|
||||
public bool EnableRelayFallback { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// If set, the connector will always use the relay endpoint and skip direct attempts.
|
||||
/// </summary>
|
||||
public bool ForceRelay { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to fetch requests (overrides HttpClient default).
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when marking fetch failures.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// Look-back period used when deriving initial published cursors.
|
||||
/// </summary>
|
||||
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
|
||||
|
||||
/// <summary>
|
||||
/// User-agent header sent with outbound requests.
|
||||
/// </summary>
|
||||
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
|
||||
|
||||
/// <summary>
|
||||
/// RSS feeds requested during fetch.
|
||||
/// </summary>
|
||||
public IList<AcscFeedOptions> Feeds { get; }
|
||||
|
||||
/// <summary>
|
||||
/// HTTP version policy requested for outbound requests.
|
||||
/// </summary>
|
||||
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
|
||||
|
||||
/// <summary>
|
||||
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
|
||||
/// </summary>
|
||||
public Version RequestVersion { get; set; } = HttpVersion.Version20;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
|
||||
}
|
||||
|
||||
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
|
||||
}
|
||||
|
||||
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (Feeds.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("At least one ACSC feed must be configured.");
|
||||
}
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
for (var i = 0; i < Feeds.Count; i++)
|
||||
{
|
||||
var feed = Feeds[i];
|
||||
feed.Validate(i);
|
||||
|
||||
if (!feed.Enabled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seen.Add(feed.Slug))
|
||||
{
|
||||
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,141 +1,141 @@
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal enum AcscEndpointPreference
|
||||
{
|
||||
Auto = 0,
|
||||
Direct = 1,
|
||||
Relay = 2,
|
||||
}
|
||||
|
||||
internal sealed record AcscCursor(
|
||||
AcscEndpointPreference PreferredEndpoint,
|
||||
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
|
||||
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static AcscCursor Empty { get; } = new(
|
||||
AcscEndpointPreference.Auto,
|
||||
EmptyFeedDictionary,
|
||||
EmptyGuidList,
|
||||
EmptyGuidList);
|
||||
|
||||
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
|
||||
=> this with { PreferredEndpoint = preference };
|
||||
|
||||
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
|
||||
{
|
||||
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
|
||||
if (values is not null)
|
||||
{
|
||||
foreach (var kvp in values)
|
||||
{
|
||||
snapshot[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
return this with { LastPublishedByFeed = snapshot };
|
||||
}
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["preferredEndpoint"] = PreferredEndpoint.ToString(),
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
var feedsDocument = new BsonDocument();
|
||||
foreach (var kvp in LastPublishedByFeed)
|
||||
{
|
||||
if (kvp.Value.HasValue)
|
||||
{
|
||||
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
|
||||
}
|
||||
}
|
||||
|
||||
document["feeds"] = feedsDocument;
|
||||
return document;
|
||||
}
|
||||
|
||||
public static AcscCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
|
||||
? ParseEndpointPreference(endpointValue.AsString)
|
||||
: AcscEndpointPreference.Auto;
|
||||
|
||||
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
|
||||
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is BsonDocument feedsDocument)
|
||||
{
|
||||
foreach (var element in feedsDocument.Elements)
|
||||
{
|
||||
feeds[element.Name] = ParseDate(element.Value);
|
||||
}
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
return new AcscCursor(
|
||||
preferredEndpoint,
|
||||
feeds,
|
||||
pendingDocuments,
|
||||
pendingMappings);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var list = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
list.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
{
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static AcscEndpointPreference ParseEndpointPreference(string? value)
|
||||
{
|
||||
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return AcscEndpointPreference.Auto;
|
||||
}
|
||||
}
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal enum AcscEndpointPreference
|
||||
{
|
||||
Auto = 0,
|
||||
Direct = 1,
|
||||
Relay = 2,
|
||||
}
|
||||
|
||||
internal sealed record AcscCursor(
|
||||
AcscEndpointPreference PreferredEndpoint,
|
||||
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
|
||||
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static AcscCursor Empty { get; } = new(
|
||||
AcscEndpointPreference.Auto,
|
||||
EmptyFeedDictionary,
|
||||
EmptyGuidList,
|
||||
EmptyGuidList);
|
||||
|
||||
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
|
||||
=> this with { PreferredEndpoint = preference };
|
||||
|
||||
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
|
||||
{
|
||||
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
|
||||
if (values is not null)
|
||||
{
|
||||
foreach (var kvp in values)
|
||||
{
|
||||
snapshot[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
return this with { LastPublishedByFeed = snapshot };
|
||||
}
|
||||
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new DocumentObject
|
||||
{
|
||||
["preferredEndpoint"] = PreferredEndpoint.ToString(),
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
var feedsDocument = new DocumentObject();
|
||||
foreach (var kvp in LastPublishedByFeed)
|
||||
{
|
||||
if (kvp.Value.HasValue)
|
||||
{
|
||||
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
|
||||
}
|
||||
}
|
||||
|
||||
document["feeds"] = feedsDocument;
|
||||
return document;
|
||||
}
|
||||
|
||||
public static AcscCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
|
||||
? ParseEndpointPreference(endpointValue.AsString)
|
||||
: AcscEndpointPreference.Auto;
|
||||
|
||||
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
|
||||
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is DocumentObject feedsDocument)
|
||||
{
|
||||
foreach (var element in feedsDocument.Elements)
|
||||
{
|
||||
feeds[element.Name] = ParseDate(element.Value);
|
||||
}
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
return new AcscCursor(
|
||||
preferredEndpoint,
|
||||
feeds,
|
||||
pendingDocuments,
|
||||
pendingMappings);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var list = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
list.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(DocumentValue value)
|
||||
{
|
||||
return value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static AcscEndpointPreference ParseEndpointPreference(string? value)
|
||||
{
|
||||
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return AcscEndpointPreference.Auto;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,97 +1,97 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
public sealed class AcscDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _fetchFallbacks;
|
||||
private readonly Counter<long> _cursorUpdates;
|
||||
private readonly Counter<long> _parseAttempts;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
|
||||
public AcscDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
|
||||
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
|
||||
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
|
||||
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
|
||||
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
|
||||
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
|
||||
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
|
||||
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
|
||||
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
|
||||
}
|
||||
|
||||
public void FetchAttempt(string feed, string mode)
|
||||
=> _fetchAttempts.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchSuccess(string feed, string mode)
|
||||
=> _fetchSuccess.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchFailure(string feed, string mode)
|
||||
=> _fetchFailures.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchUnchanged(string feed, string mode)
|
||||
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchFallback(string feed, string mode, string reason)
|
||||
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
|
||||
|
||||
public void CursorUpdated(string feed)
|
||||
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
|
||||
|
||||
public void ParseAttempt(string feed)
|
||||
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
|
||||
|
||||
public void ParseSuccess(string feed)
|
||||
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
|
||||
|
||||
public void ParseFailure(string feed, string reason)
|
||||
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
|
||||
{
|
||||
new("feed", feed),
|
||||
new("reason", reason),
|
||||
});
|
||||
|
||||
public void MapSuccess(int advisoryCount)
|
||||
{
|
||||
if (advisoryCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_mapSuccess.Add(advisoryCount);
|
||||
}
|
||||
|
||||
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
|
||||
=> new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("feed", feed),
|
||||
new KeyValuePair<string, object?>("mode", mode),
|
||||
};
|
||||
|
||||
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
|
||||
=> new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("feed", feed),
|
||||
new KeyValuePair<string, object?>("mode", mode),
|
||||
extra,
|
||||
};
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
}
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
public sealed class AcscDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _fetchFallbacks;
|
||||
private readonly Counter<long> _cursorUpdates;
|
||||
private readonly Counter<long> _parseAttempts;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
|
||||
public AcscDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
|
||||
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
|
||||
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
|
||||
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
|
||||
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
|
||||
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
|
||||
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
|
||||
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
|
||||
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
|
||||
}
|
||||
|
||||
public void FetchAttempt(string feed, string mode)
|
||||
=> _fetchAttempts.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchSuccess(string feed, string mode)
|
||||
=> _fetchSuccess.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchFailure(string feed, string mode)
|
||||
=> _fetchFailures.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchUnchanged(string feed, string mode)
|
||||
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
|
||||
|
||||
public void FetchFallback(string feed, string mode, string reason)
|
||||
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
|
||||
|
||||
public void CursorUpdated(string feed)
|
||||
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
|
||||
|
||||
public void ParseAttempt(string feed)
|
||||
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
|
||||
|
||||
public void ParseSuccess(string feed)
|
||||
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
|
||||
|
||||
public void ParseFailure(string feed, string reason)
|
||||
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
|
||||
{
|
||||
new("feed", feed),
|
||||
new("reason", reason),
|
||||
});
|
||||
|
||||
public void MapSuccess(int advisoryCount)
|
||||
{
|
||||
if (advisoryCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_mapSuccess.Add(advisoryCount);
|
||||
}
|
||||
|
||||
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
|
||||
=> new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("feed", feed),
|
||||
new KeyValuePair<string, object?>("mode", mode),
|
||||
};
|
||||
|
||||
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
|
||||
=> new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("feed", feed),
|
||||
new KeyValuePair<string, object?>("mode", mode),
|
||||
extra,
|
||||
};
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
using StellaOps.Concelier.Storage;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
|
||||
{
|
||||
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
|
||||
{
|
||||
if (document.Metadata is null)
|
||||
{
|
||||
return new AcscDocumentMetadata(string.Empty, string.Empty);
|
||||
}
|
||||
|
||||
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
|
||||
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
|
||||
return new AcscDocumentMetadata(
|
||||
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
|
||||
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
|
||||
}
|
||||
}
|
||||
using StellaOps.Concelier.Storage;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
|
||||
{
|
||||
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
|
||||
{
|
||||
if (document.Metadata is null)
|
||||
{
|
||||
return new AcscDocumentMetadata(string.Empty, string.Empty);
|
||||
}
|
||||
|
||||
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
|
||||
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
|
||||
return new AcscDocumentMetadata(
|
||||
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
|
||||
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,58 +1,58 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal sealed record AcscFeedDto(
|
||||
[property: JsonPropertyName("feedSlug")] string FeedSlug,
|
||||
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
|
||||
[property: JsonPropertyName("feedLink")] string? FeedLink,
|
||||
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
|
||||
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
|
||||
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
|
||||
{
|
||||
public static AcscFeedDto Empty { get; } = new(
|
||||
FeedSlug: string.Empty,
|
||||
FeedTitle: null,
|
||||
FeedLink: null,
|
||||
FeedUpdated: null,
|
||||
ParsedAt: DateTimeOffset.UnixEpoch,
|
||||
Entries: Array.Empty<AcscEntryDto>());
|
||||
}
|
||||
|
||||
internal sealed record AcscEntryDto(
|
||||
[property: JsonPropertyName("entryId")] string EntryId,
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("link")] string? Link,
|
||||
[property: JsonPropertyName("feedSlug")] string FeedSlug,
|
||||
[property: JsonPropertyName("published")] DateTimeOffset? Published,
|
||||
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
|
||||
[property: JsonPropertyName("summary")] string Summary,
|
||||
[property: JsonPropertyName("contentHtml")] string ContentHtml,
|
||||
[property: JsonPropertyName("contentText")] string ContentText,
|
||||
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
|
||||
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
|
||||
{
|
||||
public static AcscEntryDto Empty { get; } = new(
|
||||
EntryId: string.Empty,
|
||||
Title: string.Empty,
|
||||
Link: null,
|
||||
FeedSlug: string.Empty,
|
||||
Published: null,
|
||||
Updated: null,
|
||||
Summary: string.Empty,
|
||||
ContentHtml: string.Empty,
|
||||
ContentText: string.Empty,
|
||||
References: Array.Empty<AcscReferenceDto>(),
|
||||
Aliases: Array.Empty<string>(),
|
||||
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
internal sealed record AcscReferenceDto(
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("url")] string Url)
|
||||
{
|
||||
public static AcscReferenceDto Empty { get; } = new(
|
||||
Title: string.Empty,
|
||||
Url: string.Empty);
|
||||
}
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal sealed record AcscFeedDto(
|
||||
[property: JsonPropertyName("feedSlug")] string FeedSlug,
|
||||
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
|
||||
[property: JsonPropertyName("feedLink")] string? FeedLink,
|
||||
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
|
||||
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
|
||||
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
|
||||
{
|
||||
public static AcscFeedDto Empty { get; } = new(
|
||||
FeedSlug: string.Empty,
|
||||
FeedTitle: null,
|
||||
FeedLink: null,
|
||||
FeedUpdated: null,
|
||||
ParsedAt: DateTimeOffset.UnixEpoch,
|
||||
Entries: Array.Empty<AcscEntryDto>());
|
||||
}
|
||||
|
||||
internal sealed record AcscEntryDto(
|
||||
[property: JsonPropertyName("entryId")] string EntryId,
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("link")] string? Link,
|
||||
[property: JsonPropertyName("feedSlug")] string FeedSlug,
|
||||
[property: JsonPropertyName("published")] DateTimeOffset? Published,
|
||||
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
|
||||
[property: JsonPropertyName("summary")] string Summary,
|
||||
[property: JsonPropertyName("contentHtml")] string ContentHtml,
|
||||
[property: JsonPropertyName("contentText")] string ContentText,
|
||||
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
|
||||
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
|
||||
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
|
||||
{
|
||||
public static AcscEntryDto Empty { get; } = new(
|
||||
EntryId: string.Empty,
|
||||
Title: string.Empty,
|
||||
Link: null,
|
||||
FeedSlug: string.Empty,
|
||||
Published: null,
|
||||
Updated: null,
|
||||
Summary: string.Empty,
|
||||
ContentHtml: string.Empty,
|
||||
ContentText: string.Empty,
|
||||
References: Array.Empty<AcscReferenceDto>(),
|
||||
Aliases: Array.Empty<string>(),
|
||||
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
internal sealed record AcscReferenceDto(
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("url")] string Url)
|
||||
{
|
||||
public static AcscReferenceDto Empty { get; } = new(
|
||||
Title: string.Empty,
|
||||
Url: string.Empty);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,312 +1,312 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage;
|
||||
using StellaOps.Concelier.Storage;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal static class AcscMapper
|
||||
{
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyList<Advisory> Map(
|
||||
AcscFeedDto feed,
|
||||
DocumentRecord document,
|
||||
DtoRecord dtoRecord,
|
||||
string sourceName,
|
||||
DateTimeOffset mappedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(feed);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
ArgumentException.ThrowIfNullOrEmpty(sourceName);
|
||||
|
||||
if (feed.Entries is null || feed.Entries.Count == 0)
|
||||
{
|
||||
return Array.Empty<Advisory>();
|
||||
}
|
||||
|
||||
var advisories = new List<Advisory>(feed.Entries.Count);
|
||||
foreach (var entry in feed.Entries)
|
||||
{
|
||||
if (entry is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
|
||||
var fetchProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"document",
|
||||
document.Uri,
|
||||
document.FetchedAt.ToUniversalTime(),
|
||||
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
|
||||
|
||||
var feedProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"feed",
|
||||
feed.FeedSlug ?? string.Empty,
|
||||
feed.ParsedAt.ToUniversalTime(),
|
||||
fieldMask: new[] { "summary" });
|
||||
|
||||
var mappingProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"mapping",
|
||||
entry.EntryId ?? entry.Link ?? advisoryKey,
|
||||
mappedAt.ToUniversalTime(),
|
||||
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
fetchProvenance,
|
||||
feedProvenance,
|
||||
mappingProvenance,
|
||||
};
|
||||
|
||||
var aliases = BuildAliases(entry);
|
||||
var severity = TryGetSeverity(entry.Fields);
|
||||
var references = BuildReferences(entry, sourceName, mappedAt);
|
||||
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
|
||||
|
||||
var advisory = new Advisory(
|
||||
advisoryKey,
|
||||
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
|
||||
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
|
||||
language: "en",
|
||||
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
|
||||
modified: entry.Updated?.ToUniversalTime(),
|
||||
severity: severity,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance);
|
||||
|
||||
advisories.Add(advisory);
|
||||
}
|
||||
|
||||
return advisories;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.EntryId))
|
||||
{
|
||||
aliases.Add(entry.EntryId.Trim());
|
||||
}
|
||||
|
||||
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(alias))
|
||||
{
|
||||
aliases.Add(alias.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
|
||||
{
|
||||
var value = match.Value.ToUpperInvariant();
|
||||
aliases.Add(value);
|
||||
}
|
||||
|
||||
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
|
||||
{
|
||||
var value = match.Value.ToUpperInvariant();
|
||||
aliases.Add(value);
|
||||
}
|
||||
|
||||
return aliases.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Validation.LooksLikeHttpUrl(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!seen.Add(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
url,
|
||||
kind,
|
||||
sourceTag,
|
||||
summary,
|
||||
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
|
||||
}
|
||||
|
||||
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
|
||||
|
||||
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
|
||||
{
|
||||
if (reference is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
AddReference(reference.Url, "reference", null, reference.Title);
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<AdvisoryReference>()
|
||||
: references
|
||||
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (entry.Fields is null || entry.Fields.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(systemsAffected))
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var identifiers = systemsAffected
|
||||
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Select(static value => value.Trim())
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
if (identifiers.Length == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(identifiers.Length);
|
||||
foreach (var identifier in identifiers)
|
||||
{
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
|
||||
};
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: Array.Empty<AffectedVersionRange>(),
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance,
|
||||
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
|
||||
}
|
||||
|
||||
return packages
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
|
||||
{
|
||||
if (fields is null || fields.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var keys = new[]
|
||||
{
|
||||
"severity",
|
||||
"riskLevel",
|
||||
"threatLevel",
|
||||
"impact",
|
||||
};
|
||||
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
|
||||
{
|
||||
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
|
||||
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
|
||||
? entry.EntryId
|
||||
: !string.IsNullOrWhiteSpace(entry.Link)
|
||||
? entry.Link
|
||||
: entry.Title;
|
||||
|
||||
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
|
||||
if (string.IsNullOrEmpty(identifier))
|
||||
{
|
||||
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
return $"{sourceName}/{slug}/{identifier}";
|
||||
}
|
||||
|
||||
private static string ToSlug(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(value.Length);
|
||||
var previousDash = false;
|
||||
|
||||
foreach (var ch in value)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
builder.Append(char.ToLowerInvariant(ch));
|
||||
previousDash = false;
|
||||
}
|
||||
else if (!previousDash)
|
||||
{
|
||||
builder.Append('-');
|
||||
previousDash = true;
|
||||
}
|
||||
}
|
||||
|
||||
var slug = builder.ToString().Trim('-');
|
||||
if (string.IsNullOrEmpty(slug))
|
||||
{
|
||||
slug = CreateHash(value);
|
||||
}
|
||||
|
||||
return slug.Length <= 64 ? slug : slug[..64];
|
||||
}
|
||||
|
||||
private static string CreateHash(string value)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(value);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
|
||||
}
|
||||
}
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage;
|
||||
using StellaOps.Concelier.Storage;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc.Internal;
|
||||
|
||||
internal static class AcscMapper
|
||||
{
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyList<Advisory> Map(
|
||||
AcscFeedDto feed,
|
||||
DocumentRecord document,
|
||||
DtoRecord dtoRecord,
|
||||
string sourceName,
|
||||
DateTimeOffset mappedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(feed);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
ArgumentException.ThrowIfNullOrEmpty(sourceName);
|
||||
|
||||
if (feed.Entries is null || feed.Entries.Count == 0)
|
||||
{
|
||||
return Array.Empty<Advisory>();
|
||||
}
|
||||
|
||||
var advisories = new List<Advisory>(feed.Entries.Count);
|
||||
foreach (var entry in feed.Entries)
|
||||
{
|
||||
if (entry is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
|
||||
var fetchProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"document",
|
||||
document.Uri,
|
||||
document.FetchedAt.ToUniversalTime(),
|
||||
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
|
||||
|
||||
var feedProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"feed",
|
||||
feed.FeedSlug ?? string.Empty,
|
||||
feed.ParsedAt.ToUniversalTime(),
|
||||
fieldMask: new[] { "summary" });
|
||||
|
||||
var mappingProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"mapping",
|
||||
entry.EntryId ?? entry.Link ?? advisoryKey,
|
||||
mappedAt.ToUniversalTime(),
|
||||
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
fetchProvenance,
|
||||
feedProvenance,
|
||||
mappingProvenance,
|
||||
};
|
||||
|
||||
var aliases = BuildAliases(entry);
|
||||
var severity = TryGetSeverity(entry.Fields);
|
||||
var references = BuildReferences(entry, sourceName, mappedAt);
|
||||
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
|
||||
|
||||
var advisory = new Advisory(
|
||||
advisoryKey,
|
||||
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
|
||||
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
|
||||
language: "en",
|
||||
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
|
||||
modified: entry.Updated?.ToUniversalTime(),
|
||||
severity: severity,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance);
|
||||
|
||||
advisories.Add(advisory);
|
||||
}
|
||||
|
||||
return advisories;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(entry.EntryId))
|
||||
{
|
||||
aliases.Add(entry.EntryId.Trim());
|
||||
}
|
||||
|
||||
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(alias))
|
||||
{
|
||||
aliases.Add(alias.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
|
||||
{
|
||||
var value = match.Value.ToUpperInvariant();
|
||||
aliases.Add(value);
|
||||
}
|
||||
|
||||
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
|
||||
{
|
||||
var value = match.Value.ToUpperInvariant();
|
||||
aliases.Add(value);
|
||||
}
|
||||
|
||||
return aliases.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Validation.LooksLikeHttpUrl(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!seen.Add(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
url,
|
||||
kind,
|
||||
sourceTag,
|
||||
summary,
|
||||
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
|
||||
}
|
||||
|
||||
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
|
||||
|
||||
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
|
||||
{
|
||||
if (reference is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
AddReference(reference.Url, "reference", null, reference.Title);
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<AdvisoryReference>()
|
||||
: references
|
||||
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (entry.Fields is null || entry.Fields.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(systemsAffected))
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var identifiers = systemsAffected
|
||||
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Select(static value => value.Trim())
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
if (identifiers.Length == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(identifiers.Length);
|
||||
foreach (var identifier in identifiers)
|
||||
{
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
|
||||
};
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: Array.Empty<AffectedVersionRange>(),
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance,
|
||||
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
|
||||
}
|
||||
|
||||
return packages
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
|
||||
{
|
||||
if (fields is null || fields.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var keys = new[]
|
||||
{
|
||||
"severity",
|
||||
"riskLevel",
|
||||
"threatLevel",
|
||||
"impact",
|
||||
};
|
||||
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
|
||||
{
|
||||
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
|
||||
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
|
||||
? entry.EntryId
|
||||
: !string.IsNullOrWhiteSpace(entry.Link)
|
||||
? entry.Link
|
||||
: entry.Title;
|
||||
|
||||
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
|
||||
if (string.IsNullOrEmpty(identifier))
|
||||
{
|
||||
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
return $"{sourceName}/{slug}/{identifier}";
|
||||
}
|
||||
|
||||
private static string ToSlug(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(value.Length);
|
||||
var previousDash = false;
|
||||
|
||||
foreach (var ch in value)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
builder.Append(char.ToLowerInvariant(ch));
|
||||
previousDash = false;
|
||||
}
|
||||
else if (!previousDash)
|
||||
{
|
||||
builder.Append('-');
|
||||
previousDash = true;
|
||||
}
|
||||
}
|
||||
|
||||
var slug = builder.ToString().Trim('-');
|
||||
if (string.IsNullOrEmpty(slug))
|
||||
{
|
||||
slug = CreateHash(value);
|
||||
}
|
||||
|
||||
return slug.Length <= 64 ? slug : slug[..64];
|
||||
}
|
||||
|
||||
private static string CreateHash(string value)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(value);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,55 +1,55 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
internal static class AcscJobKinds
|
||||
{
|
||||
public const string Fetch = "source:acsc:fetch";
|
||||
public const string Parse = "source:acsc:parse";
|
||||
public const string Map = "source:acsc:map";
|
||||
public const string Probe = "source:acsc:probe";
|
||||
}
|
||||
|
||||
internal sealed class AcscFetchJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscFetchJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AcscParseJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscParseJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AcscMapJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscMapJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AcscProbeJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscProbeJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ProbeAsync(cancellationToken);
|
||||
}
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Acsc;
|
||||
|
||||
internal static class AcscJobKinds
|
||||
{
|
||||
public const string Fetch = "source:acsc:fetch";
|
||||
public const string Parse = "source:acsc:parse";
|
||||
public const string Map = "source:acsc:map";
|
||||
public const string Probe = "source:acsc:probe";
|
||||
}
|
||||
|
||||
internal sealed class AcscFetchJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscFetchJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AcscParseJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscParseJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AcscMapJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscMapJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AcscProbeJob : IJob
|
||||
{
|
||||
private readonly AcscConnector _connector;
|
||||
|
||||
public AcscProbeJob(AcscConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ProbeAsync(cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("FixtureUpdater")]
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("FixtureUpdater")]
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]
|
||||
|
||||
@@ -10,7 +10,7 @@ using System.Threading.Tasks;
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
using StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
@@ -332,7 +332,7 @@ public sealed class CccsConnector : IFeedConnector
|
||||
}
|
||||
|
||||
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
|
||||
var dtoBson = BsonDocument.Parse(dtoJson);
|
||||
var dtoBson = DocumentObject.Parse(dtoJson);
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
@@ -464,7 +464,7 @@ public sealed class CccsConnector : IFeedConnector
|
||||
|
||||
private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var document = cursor.ToDocumentObject();
|
||||
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
public sealed class CccsConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cccs";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CccsConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CccsConnector>();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
public sealed class CccsConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cccs";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CccsConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CccsConnector>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +1,50 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cccs";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCccsConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CccsFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cccs";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCccsConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CccsFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,47 +1,47 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
using StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
public static class CccsServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CccsOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0";
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
|
||||
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
|
||||
}
|
||||
|
||||
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
|
||||
clientOptions.AllowedHosts.Add("cyber.gc.ca");
|
||||
});
|
||||
|
||||
services.TryAddSingleton<HtmlContentSanitizer>();
|
||||
services.TryAddSingleton<CccsDiagnostics>();
|
||||
services.TryAddSingleton<CccsHtmlParser>();
|
||||
services.TryAddSingleton<CccsFeedClient>();
|
||||
services.AddTransient<CccsConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
using StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
public static class CccsServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CccsOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0";
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
|
||||
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
|
||||
}
|
||||
|
||||
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
|
||||
clientOptions.AllowedHosts.Add("cyber.gc.ca");
|
||||
});
|
||||
|
||||
services.TryAddSingleton<HtmlContentSanitizer>();
|
||||
services.TryAddSingleton<CccsDiagnostics>();
|
||||
services.TryAddSingleton<CccsHtmlParser>();
|
||||
services.TryAddSingleton<CccsFeedClient>();
|
||||
services.AddTransient<CccsConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,130 +1,130 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
|
||||
public sealed class CccsOptions
|
||||
{
|
||||
public const string HttpClientName = "concelier.source.cccs";
|
||||
|
||||
private readonly List<CccsFeedEndpoint> _feeds = new();
|
||||
|
||||
public CccsOptions()
|
||||
{
|
||||
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
|
||||
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Feed endpoints to poll; configure per language or content category.
|
||||
/// </summary>
|
||||
public IList<CccsFeedEndpoint> Feeds => _feeds;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of entries to enqueue per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxEntriesPerFetch { get; set; } = 80;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum remembered entries (URI+hash) for deduplication.
|
||||
/// </summary>
|
||||
public int MaxKnownEntries { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to feed and taxonomy requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Delay between successive feed requests to respect upstream throttling.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Backoff recorded in source state when fetch fails.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (_feeds.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
|
||||
}
|
||||
|
||||
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var feed in _feeds)
|
||||
{
|
||||
feed.Validate();
|
||||
if (!seenLanguages.Add(feed.Language))
|
||||
{
|
||||
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
|
||||
}
|
||||
}
|
||||
|
||||
if (MaxEntriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxKnownEntries <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
|
||||
}
|
||||
|
||||
if (FailureBackoff <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CccsFeedEndpoint
|
||||
{
|
||||
public CccsFeedEndpoint()
|
||||
{
|
||||
}
|
||||
|
||||
public CccsFeedEndpoint(string language, Uri uri)
|
||||
{
|
||||
Language = language;
|
||||
Uri = uri;
|
||||
}
|
||||
|
||||
public string Language { get; set; } = "en";
|
||||
|
||||
public Uri? Uri { get; set; }
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Language))
|
||||
{
|
||||
throw new InvalidOperationException("CCCS feed language must be specified.");
|
||||
}
|
||||
|
||||
if (Uri is null || !Uri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
|
||||
}
|
||||
}
|
||||
|
||||
public Uri BuildTaxonomyUri()
|
||||
{
|
||||
if (Uri is null)
|
||||
{
|
||||
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
|
||||
}
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
|
||||
public sealed class CccsOptions
|
||||
{
|
||||
public const string HttpClientName = "concelier.source.cccs";
|
||||
|
||||
private readonly List<CccsFeedEndpoint> _feeds = new();
|
||||
|
||||
public CccsOptions()
|
||||
{
|
||||
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
|
||||
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Feed endpoints to poll; configure per language or content category.
|
||||
/// </summary>
|
||||
public IList<CccsFeedEndpoint> Feeds => _feeds;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of entries to enqueue per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxEntriesPerFetch { get; set; } = 80;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum remembered entries (URI+hash) for deduplication.
|
||||
/// </summary>
|
||||
public int MaxKnownEntries { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to feed and taxonomy requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Delay between successive feed requests to respect upstream throttling.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Backoff recorded in source state when fetch fails.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (_feeds.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
|
||||
}
|
||||
|
||||
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var feed in _feeds)
|
||||
{
|
||||
feed.Validate();
|
||||
if (!seenLanguages.Add(feed.Language))
|
||||
{
|
||||
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
|
||||
}
|
||||
}
|
||||
|
||||
if (MaxEntriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxKnownEntries <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
|
||||
}
|
||||
|
||||
if (FailureBackoff <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CccsFeedEndpoint
|
||||
{
|
||||
public CccsFeedEndpoint()
|
||||
{
|
||||
}
|
||||
|
||||
public CccsFeedEndpoint(string language, Uri uri)
|
||||
{
|
||||
Language = language;
|
||||
Uri = uri;
|
||||
}
|
||||
|
||||
public string Language { get; set; } = "en";
|
||||
|
||||
public Uri? Uri { get; set; }
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Language))
|
||||
{
|
||||
throw new InvalidOperationException("CCCS feed language must be specified.");
|
||||
}
|
||||
|
||||
if (Uri is null || !Uri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
|
||||
}
|
||||
}
|
||||
|
||||
public Uri BuildTaxonomyUri()
|
||||
{
|
||||
if (Uri is null)
|
||||
{
|
||||
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
|
||||
}
|
||||
|
||||
var language = Uri.GetQueryParameterValueOrDefault("lang", Language);
|
||||
var taxonomyBuilder = new UriBuilder(Uri)
|
||||
{
|
||||
@@ -135,46 +135,46 @@ public sealed class CccsFeedEndpoint
|
||||
return taxonomyBuilder.Uri;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class CccsUriExtensions
|
||||
{
|
||||
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
|
||||
{
|
||||
if (uri is null)
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var query = uri.Query;
|
||||
if (string.IsNullOrEmpty(query))
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
|
||||
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separatorIndex = pair.IndexOf('=');
|
||||
if (separatorIndex < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var left = pair[..separatorIndex].Trim();
|
||||
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var right = pair[(separatorIndex + 1)..].Trim();
|
||||
if (right.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return Uri.UnescapeDataString(right);
|
||||
}
|
||||
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class CccsUriExtensions
|
||||
{
|
||||
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
|
||||
{
|
||||
if (uri is null)
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var query = uri.Query;
|
||||
if (string.IsNullOrEmpty(query))
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
|
||||
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separatorIndex = pair.IndexOf('=');
|
||||
if (separatorIndex < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var left = pair[..separatorIndex].Trim();
|
||||
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var right = pair[(separatorIndex + 1)..].Trim();
|
||||
if (right.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return Uri.UnescapeDataString(right);
|
||||
}
|
||||
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsAdvisoryDto
|
||||
{
|
||||
[JsonPropertyName("sourceId")]
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("serialNumber")]
|
||||
public string SerialNumber { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "en";
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("canonicalUrl")]
|
||||
public string CanonicalUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("contentHtml")]
|
||||
public string ContentHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("alertType")]
|
||||
public string? AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
internal sealed record CccsReferenceDto(
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("label")] string? Label);
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsAdvisoryDto
|
||||
{
|
||||
[JsonPropertyName("sourceId")]
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("serialNumber")]
|
||||
public string SerialNumber { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "en";
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("canonicalUrl")]
|
||||
public string CanonicalUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("contentHtml")]
|
||||
public string ContentHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("alertType")]
|
||||
public string? AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
internal sealed record CccsReferenceDto(
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("label")] string? Label);
|
||||
|
||||
@@ -1,145 +1,145 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, string> KnownEntryHashes,
|
||||
DateTimeOffset? LastFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
|
||||
|
||||
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
{
|
||||
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
|
||||
return this with { PendingDocuments = distinct };
|
||||
}
|
||||
|
||||
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
{
|
||||
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
|
||||
return this with { PendingMappings = distinct };
|
||||
}
|
||||
|
||||
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
|
||||
{
|
||||
var map = hashes is null || hashes.Count == 0
|
||||
? EmptyHashes
|
||||
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
|
||||
return this with { KnownEntryHashes = map };
|
||||
}
|
||||
|
||||
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var doc = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (KnownEntryHashes.Count > 0)
|
||||
{
|
||||
var hashes = new BsonArray();
|
||||
foreach (var kvp in KnownEntryHashes)
|
||||
{
|
||||
hashes.Add(new BsonDocument
|
||||
{
|
||||
["uri"] = kvp.Key,
|
||||
["hash"] = kvp.Value,
|
||||
});
|
||||
}
|
||||
|
||||
doc["knownEntryHashes"] = hashes;
|
||||
}
|
||||
|
||||
if (LastFetchAt.HasValue)
|
||||
{
|
||||
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
public static CccsCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var hashes = ReadHashMap(document);
|
||||
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
|
||||
? ParseDateTime(value)
|
||||
: null;
|
||||
|
||||
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidCollection;
|
||||
}
|
||||
|
||||
var items = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
items.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> ReadHashMap(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyHashes;
|
||||
}
|
||||
|
||||
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is not BsonDocument entry)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull
|
||||
? hashValue.AsString
|
||||
: string.Empty;
|
||||
map[uriValue.AsString] = hash;
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDateTime(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, string> KnownEntryHashes,
|
||||
DateTimeOffset? LastFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
|
||||
|
||||
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
{
|
||||
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
|
||||
return this with { PendingDocuments = distinct };
|
||||
}
|
||||
|
||||
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
{
|
||||
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
|
||||
return this with { PendingMappings = distinct };
|
||||
}
|
||||
|
||||
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
|
||||
{
|
||||
var map = hashes is null || hashes.Count == 0
|
||||
? EmptyHashes
|
||||
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
|
||||
return this with { KnownEntryHashes = map };
|
||||
}
|
||||
|
||||
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastFetchAt = timestamp };
|
||||
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var doc = new DocumentObject
|
||||
{
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (KnownEntryHashes.Count > 0)
|
||||
{
|
||||
var hashes = new DocumentArray();
|
||||
foreach (var kvp in KnownEntryHashes)
|
||||
{
|
||||
hashes.Add(new DocumentObject
|
||||
{
|
||||
["uri"] = kvp.Key,
|
||||
["hash"] = kvp.Value,
|
||||
});
|
||||
}
|
||||
|
||||
doc["knownEntryHashes"] = hashes;
|
||||
}
|
||||
|
||||
if (LastFetchAt.HasValue)
|
||||
{
|
||||
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
public static CccsCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var hashes = ReadHashMap(document);
|
||||
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
|
||||
? ParseDateTime(value)
|
||||
: null;
|
||||
|
||||
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyGuidCollection;
|
||||
}
|
||||
|
||||
var items = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
items.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> ReadHashMap(DocumentObject document)
|
||||
{
|
||||
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not DocumentArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyHashes;
|
||||
}
|
||||
|
||||
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is not DocumentObject entry)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsDocumentNull || string.IsNullOrWhiteSpace(uriValue.AsString))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsDocumentNull
|
||||
? hashValue.AsString
|
||||
: string.Empty;
|
||||
map[uriValue.AsString] = hash;
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDateTime(DocumentValue value)
|
||||
=> value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,58 +1,58 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
public sealed class CccsDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Cccs";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchDocuments;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _parseQuarantine;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
|
||||
public CccsDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
|
||||
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
|
||||
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
|
||||
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
|
||||
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
|
||||
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
|
||||
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
|
||||
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
|
||||
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
|
||||
}
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchSuccess() => _fetchSuccess.Add(1);
|
||||
|
||||
public void FetchDocument() => _fetchDocuments.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void ParseSuccess() => _parseSuccess.Add(1);
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void ParseQuarantine() => _parseQuarantine.Add(1);
|
||||
|
||||
public void MapSuccess() => _mapSuccess.Add(1);
|
||||
|
||||
public void MapFailure() => _mapFailures.Add(1);
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
public sealed class CccsDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Cccs";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchDocuments;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _parseQuarantine;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
|
||||
public CccsDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
|
||||
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
|
||||
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
|
||||
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
|
||||
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
|
||||
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
|
||||
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
|
||||
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
|
||||
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
|
||||
}
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchSuccess() => _fetchSuccess.Add(1);
|
||||
|
||||
public void FetchDocument() => _fetchDocuments.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void ParseSuccess() => _parseSuccess.Add(1);
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void ParseQuarantine() => _parseQuarantine.Add(1);
|
||||
|
||||
public void MapSuccess() => _mapSuccess.Add(1);
|
||||
|
||||
public void MapFailure() => _mapFailures.Add(1);
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
|
||||
@@ -1,146 +1,146 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
public sealed class CccsFeedClient
|
||||
{
|
||||
private static readonly string[] AcceptHeaders =
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.api+json;q=0.9",
|
||||
"text/json;q=0.8",
|
||||
"application/*+json;q=0.7",
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly ILogger<CccsFeedClient> _logger;
|
||||
|
||||
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(endpoint);
|
||||
if (endpoint.Uri is null)
|
||||
{
|
||||
throw new InvalidOperationException("Feed endpoint URI must be configured.");
|
||||
}
|
||||
|
||||
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
|
||||
{
|
||||
AcceptHeaders = AcceptHeaders,
|
||||
TimeoutOverride = requestTimeout,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = endpoint.Language,
|
||||
["cccs.feedUri"] = endpoint.Uri.ToString(),
|
||||
},
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Content is null)
|
||||
{
|
||||
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
|
||||
return CccsFeedResult.Empty;
|
||||
}
|
||||
|
||||
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
|
||||
if (feedResponse is null || feedResponse.Error)
|
||||
{
|
||||
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
|
||||
return CccsFeedResult.Empty;
|
||||
}
|
||||
|
||||
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
|
||||
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
|
||||
return new CccsFeedResult(items, taxonomy, result.LastModified);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
|
||||
{
|
||||
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
var taxonomyUri = endpoint.BuildTaxonomyUri();
|
||||
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
|
||||
{
|
||||
AcceptHeaders = AcceptHeaders,
|
||||
TimeoutOverride = timeout,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = endpoint.Language,
|
||||
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
|
||||
},
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (!result.IsSuccess || result.Content is null)
|
||||
{
|
||||
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
|
||||
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
|
||||
if (taxonomyResponse is null || taxonomyResponse.Error)
|
||||
{
|
||||
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
|
||||
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
|
||||
foreach (var item in taxonomyResponse.Response)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(item.Title))
|
||||
{
|
||||
map[item.Id] = item.Title!;
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
}
|
||||
|
||||
private static T? Deserialize<T>(byte[] content)
|
||||
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
public sealed class CccsFeedClient
|
||||
{
|
||||
private static readonly string[] AcceptHeaders =
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.api+json;q=0.9",
|
||||
"text/json;q=0.8",
|
||||
"application/*+json;q=0.7",
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly ILogger<CccsFeedClient> _logger;
|
||||
|
||||
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(endpoint);
|
||||
if (endpoint.Uri is null)
|
||||
{
|
||||
throw new InvalidOperationException("Feed endpoint URI must be configured.");
|
||||
}
|
||||
|
||||
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
|
||||
{
|
||||
AcceptHeaders = AcceptHeaders,
|
||||
TimeoutOverride = requestTimeout,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = endpoint.Language,
|
||||
["cccs.feedUri"] = endpoint.Uri.ToString(),
|
||||
},
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Content is null)
|
||||
{
|
||||
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
|
||||
return CccsFeedResult.Empty;
|
||||
}
|
||||
|
||||
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
|
||||
if (feedResponse is null || feedResponse.Error)
|
||||
{
|
||||
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
|
||||
return CccsFeedResult.Empty;
|
||||
}
|
||||
|
||||
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
|
||||
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
|
||||
return new CccsFeedResult(items, taxonomy, result.LastModified);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
|
||||
{
|
||||
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
var taxonomyUri = endpoint.BuildTaxonomyUri();
|
||||
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
|
||||
{
|
||||
AcceptHeaders = AcceptHeaders,
|
||||
TimeoutOverride = timeout,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = endpoint.Language,
|
||||
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
|
||||
},
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (!result.IsSuccess || result.Content is null)
|
||||
{
|
||||
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
|
||||
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
|
||||
if (taxonomyResponse is null || taxonomyResponse.Error)
|
||||
{
|
||||
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
|
||||
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
|
||||
foreach (var item in taxonomyResponse.Response)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(item.Title))
|
||||
{
|
||||
map[item.Id] = item.Title!;
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
}
|
||||
|
||||
private static T? Deserialize<T>(byte[] content)
|
||||
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
|
||||
}
|
||||
|
||||
@@ -1,101 +1,101 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed class CccsFeedResponse
|
||||
{
|
||||
[JsonPropertyName("ERROR")]
|
||||
public bool Error { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public List<CccsFeedItem> Response { get; init; } = new();
|
||||
}
|
||||
|
||||
internal sealed class CccsFeedItem
|
||||
{
|
||||
[JsonPropertyName("nid")]
|
||||
public int Nid { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("uuid")]
|
||||
public string? Uuid { get; init; }
|
||||
|
||||
[JsonPropertyName("banner")]
|
||||
public string? Banner { get; init; }
|
||||
|
||||
[JsonPropertyName("lang")]
|
||||
public string? Language { get; init; }
|
||||
|
||||
[JsonPropertyName("date_modified")]
|
||||
public string? DateModified { get; init; }
|
||||
|
||||
[JsonPropertyName("date_modified_ts")]
|
||||
public string? DateModifiedTimestamp { get; init; }
|
||||
|
||||
[JsonPropertyName("date_created")]
|
||||
public string? DateCreated { get; init; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("body")]
|
||||
public string[] Body { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; init; }
|
||||
|
||||
[JsonPropertyName("alert_type")]
|
||||
public JsonElement AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("serial_number")]
|
||||
public string? SerialNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("moderation_state")]
|
||||
public string? ModerationState { get; init; }
|
||||
|
||||
[JsonPropertyName("external_url")]
|
||||
public string? ExternalUrl { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class CccsTaxonomyResponse
|
||||
{
|
||||
[JsonPropertyName("ERROR")]
|
||||
public bool Error { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public List<CccsTaxonomyItem> Response { get; init; } = new();
|
||||
}
|
||||
|
||||
internal sealed class CccsTaxonomyItem
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public int Id { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CccsFeedResult(
|
||||
IReadOnlyList<CccsFeedItem> Items,
|
||||
IReadOnlyDictionary<int, string> AlertTypes,
|
||||
DateTimeOffset? LastModifiedUtc)
|
||||
{
|
||||
public static CccsFeedResult Empty { get; } = new(
|
||||
Array.Empty<CccsFeedItem>(),
|
||||
new Dictionary<int, string>(0),
|
||||
null);
|
||||
}
|
||||
|
||||
internal static class CccsFeedResultExtensions
|
||||
{
|
||||
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
|
||||
=> new(items, alertTypes, lastModified);
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed class CccsFeedResponse
|
||||
{
|
||||
[JsonPropertyName("ERROR")]
|
||||
public bool Error { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public List<CccsFeedItem> Response { get; init; } = new();
|
||||
}
|
||||
|
||||
internal sealed class CccsFeedItem
|
||||
{
|
||||
[JsonPropertyName("nid")]
|
||||
public int Nid { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("uuid")]
|
||||
public string? Uuid { get; init; }
|
||||
|
||||
[JsonPropertyName("banner")]
|
||||
public string? Banner { get; init; }
|
||||
|
||||
[JsonPropertyName("lang")]
|
||||
public string? Language { get; init; }
|
||||
|
||||
[JsonPropertyName("date_modified")]
|
||||
public string? DateModified { get; init; }
|
||||
|
||||
[JsonPropertyName("date_modified_ts")]
|
||||
public string? DateModifiedTimestamp { get; init; }
|
||||
|
||||
[JsonPropertyName("date_created")]
|
||||
public string? DateCreated { get; init; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("body")]
|
||||
public string[] Body { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; init; }
|
||||
|
||||
[JsonPropertyName("alert_type")]
|
||||
public JsonElement AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("serial_number")]
|
||||
public string? SerialNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("moderation_state")]
|
||||
public string? ModerationState { get; init; }
|
||||
|
||||
[JsonPropertyName("external_url")]
|
||||
public string? ExternalUrl { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class CccsTaxonomyResponse
|
||||
{
|
||||
[JsonPropertyName("ERROR")]
|
||||
public bool Error { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public List<CccsTaxonomyItem> Response { get; init; } = new();
|
||||
}
|
||||
|
||||
internal sealed class CccsTaxonomyItem
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public int Id { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CccsFeedResult(
|
||||
IReadOnlyList<CccsFeedItem> Items,
|
||||
IReadOnlyDictionary<int, string> AlertTypes,
|
||||
DateTimeOffset? LastModifiedUtc)
|
||||
{
|
||||
public static CccsFeedResult Empty { get; } = new(
|
||||
Array.Empty<CccsFeedItem>(),
|
||||
new Dictionary<int, string>(0),
|
||||
null);
|
||||
}
|
||||
|
||||
internal static class CccsFeedResultExtensions
|
||||
{
|
||||
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
|
||||
=> new(items, alertTypes, lastModified);
|
||||
}
|
||||
|
||||
@@ -1,353 +1,353 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
public sealed class CccsHtmlParser
|
||||
{
|
||||
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[::]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[::]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
|
||||
|
||||
private static readonly CultureInfo[] EnglishCultures =
|
||||
{
|
||||
CultureInfo.GetCultureInfo("en-CA"),
|
||||
CultureInfo.GetCultureInfo("en-US"),
|
||||
CultureInfo.InvariantCulture,
|
||||
};
|
||||
|
||||
private static readonly CultureInfo[] FrenchCultures =
|
||||
{
|
||||
CultureInfo.GetCultureInfo("fr-CA"),
|
||||
CultureInfo.GetCultureInfo("fr-FR"),
|
||||
CultureInfo.InvariantCulture,
|
||||
};
|
||||
|
||||
private static readonly string[] ProductHeadingKeywords =
|
||||
{
|
||||
"affected",
|
||||
"produit",
|
||||
"produits",
|
||||
"produits touch",
|
||||
"produits concern",
|
||||
"mesures recommand",
|
||||
};
|
||||
|
||||
private static readonly string[] TrackingParameterPrefixes =
|
||||
{
|
||||
"utm_",
|
||||
"mc_",
|
||||
"mkt_",
|
||||
"elq",
|
||||
};
|
||||
|
||||
private readonly HtmlContentSanitizer _sanitizer;
|
||||
private readonly HtmlParser _parser;
|
||||
|
||||
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
|
||||
{
|
||||
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
|
||||
_parser = new HtmlParser(new HtmlParserOptions
|
||||
{
|
||||
IsScripting = false,
|
||||
IsKeepingSourceReferences = false,
|
||||
});
|
||||
}
|
||||
|
||||
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(raw);
|
||||
|
||||
var baseUri = TryCreateUri(raw.CanonicalUrl);
|
||||
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
|
||||
var body = document.Body ?? document.DocumentElement;
|
||||
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
|
||||
var contentRoot = body ?? document.DocumentElement;
|
||||
|
||||
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
|
||||
? raw.SerialNumber!.Trim()
|
||||
: ExtractSerialNumber(document) ?? raw.SourceId;
|
||||
|
||||
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
|
||||
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
|
||||
var products = ExtractProducts(contentRoot);
|
||||
var cveIds = ExtractCveIds(document);
|
||||
|
||||
return new CccsAdvisoryDto
|
||||
{
|
||||
SourceId = raw.SourceId,
|
||||
SerialNumber = serialNumber,
|
||||
Language = raw.Language,
|
||||
Title = raw.Title,
|
||||
Summary = CollapseWhitespace(raw.Summary),
|
||||
CanonicalUrl = raw.CanonicalUrl,
|
||||
ContentHtml = sanitized,
|
||||
Published = published,
|
||||
Modified = raw.Modified ?? published,
|
||||
AlertType = raw.AlertType,
|
||||
Subject = raw.Subject,
|
||||
Products = products,
|
||||
References = references,
|
||||
CveIds = cveIds,
|
||||
};
|
||||
}
|
||||
|
||||
private static Uri? TryCreateUri(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
|
||||
}
|
||||
|
||||
private static string? ExtractSerialNumber(IDocument document)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var element in document.QuerySelectorAll("strong, p, div"))
|
||||
{
|
||||
var text = element.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = SerialRegex.Match(text);
|
||||
if (match.Success && match.Groups["id"].Success)
|
||||
{
|
||||
var value = match.Groups["id"].Value.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var bodyText = document.Body.TextContent;
|
||||
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
|
||||
return fallback.Success && fallback.Groups["id"].Success
|
||||
? fallback.Groups["id"].Value.Trim()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ExtractDate(IDocument document, string language)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var textSegments = new List<string>();
|
||||
foreach (var element in document.QuerySelectorAll("strong, p, div"))
|
||||
{
|
||||
var text = element.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = DateRegex.Match(text);
|
||||
if (match.Success && match.Groups["date"].Success)
|
||||
{
|
||||
textSegments.Add(match.Groups["date"].Value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
|
||||
{
|
||||
textSegments.Add(document.Body.TextContent);
|
||||
}
|
||||
|
||||
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
|
||||
|
||||
foreach (var segment in textSegments)
|
||||
{
|
||||
foreach (var culture in cultures)
|
||||
{
|
||||
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return new DateTimeOffset(parsed.ToUniversalTime());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractProducts(IElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var results = new List<string>();
|
||||
|
||||
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
|
||||
{
|
||||
var text = heading.TextContent?.Trim();
|
||||
if (!IsProductHeading(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sibling = heading.NextElementSibling;
|
||||
while (sibling is not null)
|
||||
{
|
||||
if (IsHeading(sibling))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (IsListElement(sibling))
|
||||
{
|
||||
AppendListItems(sibling, results);
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (IsContentContainer(sibling))
|
||||
{
|
||||
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
|
||||
{
|
||||
AppendListItems(list, results);
|
||||
}
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sibling = sibling.NextElementSibling;
|
||||
}
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (results.Count == 0)
|
||||
{
|
||||
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
|
||||
{
|
||||
var itemText = CollapseWhitespace(li.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(itemText))
|
||||
{
|
||||
results.Add(itemText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: results
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool IsProductHeading(string? heading)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(heading))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var lowered = heading.ToLowerInvariant();
|
||||
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static bool IsHeading(IElement element)
|
||||
=> element.LocalName.Length == 2
|
||||
&& element.LocalName[0] == 'h'
|
||||
&& char.IsDigit(element.LocalName[1]);
|
||||
|
||||
private static bool IsListElement(IElement element)
|
||||
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsContentContainer(IElement element)
|
||||
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
|
||||
{
|
||||
foreach (var li in listElement.QuerySelectorAll("li"))
|
||||
{
|
||||
if (li is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var clone = li.Clone(true) as IElement;
|
||||
if (clone is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
|
||||
{
|
||||
nested.Remove();
|
||||
}
|
||||
|
||||
var itemText = CollapseWhitespace(clone.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(itemText))
|
||||
{
|
||||
buffer.Add(itemText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return Array.Empty<CccsReferenceDto>();
|
||||
}
|
||||
|
||||
var references = new List<CccsReferenceDto>();
|
||||
foreach (var anchor in root.QuerySelectorAll("a[href]"))
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
var normalized = NormalizeReferenceUrl(href, baseUri, language);
|
||||
if (normalized is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var label = CollapseWhitespace(anchor.TextContent);
|
||||
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<CccsReferenceDto>()
|
||||
: references
|
||||
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
|
||||
.Select(group => group.First())
|
||||
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
public sealed class CccsHtmlParser
|
||||
{
|
||||
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[::]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[::]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
|
||||
|
||||
private static readonly CultureInfo[] EnglishCultures =
|
||||
{
|
||||
CultureInfo.GetCultureInfo("en-CA"),
|
||||
CultureInfo.GetCultureInfo("en-US"),
|
||||
CultureInfo.InvariantCulture,
|
||||
};
|
||||
|
||||
private static readonly CultureInfo[] FrenchCultures =
|
||||
{
|
||||
CultureInfo.GetCultureInfo("fr-CA"),
|
||||
CultureInfo.GetCultureInfo("fr-FR"),
|
||||
CultureInfo.InvariantCulture,
|
||||
};
|
||||
|
||||
private static readonly string[] ProductHeadingKeywords =
|
||||
{
|
||||
"affected",
|
||||
"produit",
|
||||
"produits",
|
||||
"produits touch",
|
||||
"produits concern",
|
||||
"mesures recommand",
|
||||
};
|
||||
|
||||
private static readonly string[] TrackingParameterPrefixes =
|
||||
{
|
||||
"utm_",
|
||||
"mc_",
|
||||
"mkt_",
|
||||
"elq",
|
||||
};
|
||||
|
||||
private readonly HtmlContentSanitizer _sanitizer;
|
||||
private readonly HtmlParser _parser;
|
||||
|
||||
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
|
||||
{
|
||||
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
|
||||
_parser = new HtmlParser(new HtmlParserOptions
|
||||
{
|
||||
IsScripting = false,
|
||||
IsKeepingSourceReferences = false,
|
||||
});
|
||||
}
|
||||
|
||||
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(raw);
|
||||
|
||||
var baseUri = TryCreateUri(raw.CanonicalUrl);
|
||||
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
|
||||
var body = document.Body ?? document.DocumentElement;
|
||||
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
|
||||
var contentRoot = body ?? document.DocumentElement;
|
||||
|
||||
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
|
||||
? raw.SerialNumber!.Trim()
|
||||
: ExtractSerialNumber(document) ?? raw.SourceId;
|
||||
|
||||
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
|
||||
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
|
||||
var products = ExtractProducts(contentRoot);
|
||||
var cveIds = ExtractCveIds(document);
|
||||
|
||||
return new CccsAdvisoryDto
|
||||
{
|
||||
SourceId = raw.SourceId,
|
||||
SerialNumber = serialNumber,
|
||||
Language = raw.Language,
|
||||
Title = raw.Title,
|
||||
Summary = CollapseWhitespace(raw.Summary),
|
||||
CanonicalUrl = raw.CanonicalUrl,
|
||||
ContentHtml = sanitized,
|
||||
Published = published,
|
||||
Modified = raw.Modified ?? published,
|
||||
AlertType = raw.AlertType,
|
||||
Subject = raw.Subject,
|
||||
Products = products,
|
||||
References = references,
|
||||
CveIds = cveIds,
|
||||
};
|
||||
}
|
||||
|
||||
private static Uri? TryCreateUri(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
|
||||
}
|
||||
|
||||
private static string? ExtractSerialNumber(IDocument document)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var element in document.QuerySelectorAll("strong, p, div"))
|
||||
{
|
||||
var text = element.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = SerialRegex.Match(text);
|
||||
if (match.Success && match.Groups["id"].Success)
|
||||
{
|
||||
var value = match.Groups["id"].Value.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var bodyText = document.Body.TextContent;
|
||||
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
|
||||
return fallback.Success && fallback.Groups["id"].Success
|
||||
? fallback.Groups["id"].Value.Trim()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ExtractDate(IDocument document, string language)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var textSegments = new List<string>();
|
||||
foreach (var element in document.QuerySelectorAll("strong, p, div"))
|
||||
{
|
||||
var text = element.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = DateRegex.Match(text);
|
||||
if (match.Success && match.Groups["date"].Success)
|
||||
{
|
||||
textSegments.Add(match.Groups["date"].Value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
|
||||
{
|
||||
textSegments.Add(document.Body.TextContent);
|
||||
}
|
||||
|
||||
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
|
||||
|
||||
foreach (var segment in textSegments)
|
||||
{
|
||||
foreach (var culture in cultures)
|
||||
{
|
||||
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return new DateTimeOffset(parsed.ToUniversalTime());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractProducts(IElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var results = new List<string>();
|
||||
|
||||
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
|
||||
{
|
||||
var text = heading.TextContent?.Trim();
|
||||
if (!IsProductHeading(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sibling = heading.NextElementSibling;
|
||||
while (sibling is not null)
|
||||
{
|
||||
if (IsHeading(sibling))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (IsListElement(sibling))
|
||||
{
|
||||
AppendListItems(sibling, results);
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (IsContentContainer(sibling))
|
||||
{
|
||||
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
|
||||
{
|
||||
AppendListItems(list, results);
|
||||
}
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sibling = sibling.NextElementSibling;
|
||||
}
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (results.Count == 0)
|
||||
{
|
||||
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
|
||||
{
|
||||
var itemText = CollapseWhitespace(li.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(itemText))
|
||||
{
|
||||
results.Add(itemText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: results
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool IsProductHeading(string? heading)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(heading))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var lowered = heading.ToLowerInvariant();
|
||||
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static bool IsHeading(IElement element)
|
||||
=> element.LocalName.Length == 2
|
||||
&& element.LocalName[0] == 'h'
|
||||
&& char.IsDigit(element.LocalName[1]);
|
||||
|
||||
private static bool IsListElement(IElement element)
|
||||
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsContentContainer(IElement element)
|
||||
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
|
||||
{
|
||||
foreach (var li in listElement.QuerySelectorAll("li"))
|
||||
{
|
||||
if (li is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var clone = li.Clone(true) as IElement;
|
||||
if (clone is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
|
||||
{
|
||||
nested.Remove();
|
||||
}
|
||||
|
||||
var itemText = CollapseWhitespace(clone.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(itemText))
|
||||
{
|
||||
buffer.Add(itemText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return Array.Empty<CccsReferenceDto>();
|
||||
}
|
||||
|
||||
var references = new List<CccsReferenceDto>();
|
||||
foreach (var anchor in root.QuerySelectorAll("a[href]"))
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
var normalized = NormalizeReferenceUrl(href, baseUri, language);
|
||||
if (normalized is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var label = CollapseWhitespace(anchor.TextContent);
|
||||
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<CccsReferenceDto>()
|
||||
: references
|
||||
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
|
||||
.Select(group => group.First())
|
||||
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
return null;
|
||||
@@ -363,89 +363,89 @@ public sealed class CccsHtmlParser
|
||||
}
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(absolute)
|
||||
{
|
||||
Fragment = string.Empty,
|
||||
};
|
||||
|
||||
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
|
||||
builder.Query = filteredQuery;
|
||||
|
||||
return builder.Uri.ToString();
|
||||
}
|
||||
|
||||
private static string FilterTrackingParameters(string query, Uri uri, string language)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = query.TrimStart('?');
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
|
||||
var kept = new List<string>();
|
||||
|
||||
foreach (var parameter in parameters)
|
||||
{
|
||||
var separatorIndex = parameter.IndexOf('=');
|
||||
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
|
||||
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
|
||||
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
kept.Add($"lang={language}");
|
||||
continue;
|
||||
}
|
||||
|
||||
kept.Add(parameter);
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
|
||||
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
kept.Add($"lang={language}");
|
||||
}
|
||||
|
||||
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return matches
|
||||
.Select(match => match.Value.ToUpperInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(value => value, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? CollapseWhitespace(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
|
||||
return collapsed.Length == 0 ? null : collapsed;
|
||||
}
|
||||
}
|
||||
var builder = new UriBuilder(absolute)
|
||||
{
|
||||
Fragment = string.Empty,
|
||||
};
|
||||
|
||||
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
|
||||
builder.Query = filteredQuery;
|
||||
|
||||
return builder.Uri.ToString();
|
||||
}
|
||||
|
||||
private static string FilterTrackingParameters(string query, Uri uri, string language)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = query.TrimStart('?');
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
|
||||
var kept = new List<string>();
|
||||
|
||||
foreach (var parameter in parameters)
|
||||
{
|
||||
var separatorIndex = parameter.IndexOf('=');
|
||||
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
|
||||
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
|
||||
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
kept.Add($"lang={language}");
|
||||
continue;
|
||||
}
|
||||
|
||||
kept.Add(parameter);
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
|
||||
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
kept.Add($"lang={language}");
|
||||
}
|
||||
|
||||
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return matches
|
||||
.Select(match => match.Value.ToUpperInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(value => value, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? CollapseWhitespace(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
|
||||
return collapsed.Length == 0 ? null : collapsed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,258 +1,258 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage;
|
||||
using StellaOps.Concelier.Normalization.SemVer;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal static class CccsMapper
|
||||
{
|
||||
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AlertType ?? dto.SerialNumber,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory })
|
||||
};
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.SerialNumber,
|
||||
title: dto.Title,
|
||||
summary: dto.Summary,
|
||||
language: dto.Language,
|
||||
published: dto.Published ?? dto.Modified,
|
||||
modified: dto.Modified ?? dto.Published,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new List<string>(capacity: 4)
|
||||
{
|
||||
dto.SerialNumber,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.SourceId)
|
||||
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
aliases.Add(dto.SourceId);
|
||||
}
|
||||
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>
|
||||
{
|
||||
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
dto.CanonicalUrl,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }))
|
||||
};
|
||||
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
"reference",
|
||||
"cccs",
|
||||
reference.Label,
|
||||
new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
return references
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Products.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Products.Count);
|
||||
for (var index = 0; index < dto.Products.Count; index++)
|
||||
{
|
||||
var product = dto.Products[index];
|
||||
if (string.IsNullOrWhiteSpace(product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var identifier = product.Trim();
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var rangeAnchor = $"cccs:{dto.SerialNumber}:{index}";
|
||||
var versionRanges = BuildVersionRanges(product, rangeAnchor, recordedAt);
|
||||
var normalizedVersions = BuildNormalizedVersions(versionRanges, rangeAnchor);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: versionRanges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: normalizedVersions));
|
||||
}
|
||||
|
||||
return packages.Count == 0
|
||||
? Array.Empty<AffectedPackage>()
|
||||
: packages
|
||||
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(string productText, string rangeAnchor, DateTimeOffset recordedAt)
|
||||
{
|
||||
var versionText = ExtractFirstVersionToken(productText);
|
||||
if (string.IsNullOrWhiteSpace(versionText))
|
||||
{
|
||||
return Array.Empty<AffectedVersionRange>();
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"range",
|
||||
rangeAnchor,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges });
|
||||
|
||||
var vendorExtensions = new Dictionary<string, string>
|
||||
{
|
||||
["cccs.version.raw"] = versionText!,
|
||||
["cccs.anchor"] = rangeAnchor,
|
||||
};
|
||||
|
||||
var semVerResults = SemVerRangeRuleBuilder.Build(versionText!, patchedVersion: null, provenanceNote: rangeAnchor);
|
||||
if (semVerResults.Count > 0)
|
||||
{
|
||||
return semVerResults.Select(result =>
|
||||
new AffectedVersionRange(
|
||||
rangeKind: NormalizedVersionSchemes.SemVer,
|
||||
introducedVersion: result.Primitive.Introduced,
|
||||
fixedVersion: result.Primitive.Fixed,
|
||||
lastAffectedVersion: result.Primitive.LastAffected,
|
||||
rangeExpression: result.Expression ?? versionText!,
|
||||
provenance: provenance,
|
||||
primitives: new RangePrimitives(
|
||||
result.Primitive,
|
||||
Nevra: null,
|
||||
Evr: null,
|
||||
VendorExtensions: vendorExtensions)))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
var primitives = new RangePrimitives(
|
||||
new SemVerPrimitive(
|
||||
Introduced: versionText,
|
||||
IntroducedInclusive: true,
|
||||
Fixed: null,
|
||||
FixedInclusive: false,
|
||||
LastAffected: null,
|
||||
LastAffectedInclusive: true,
|
||||
ConstraintExpression: null,
|
||||
ExactValue: versionText),
|
||||
Nevra: null,
|
||||
Evr: null,
|
||||
VendorExtensions: vendorExtensions);
|
||||
|
||||
return new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: NormalizedVersionSchemes.SemVer,
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: versionText,
|
||||
provenance: provenance,
|
||||
primitives: primitives),
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
IReadOnlyList<AffectedVersionRange> ranges,
|
||||
string rangeAnchor)
|
||||
{
|
||||
if (ranges.Count == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var rules = new List<NormalizedVersionRule>(ranges.Count);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var rule = range.ToNormalizedVersionRule(rangeAnchor);
|
||||
if (rule is not null)
|
||||
{
|
||||
rules.Add(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
|
||||
}
|
||||
|
||||
private static string? ExtractFirstVersionToken(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var match = Regex.Match(value, @"\d+(?:\.\d+){0,3}(?:[A-Za-z0-9\-_]*)?");
|
||||
return match.Success ? match.Value : null;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage;
|
||||
using StellaOps.Concelier.Normalization.SemVer;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal static class CccsMapper
|
||||
{
|
||||
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AlertType ?? dto.SerialNumber,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory })
|
||||
};
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.SerialNumber,
|
||||
title: dto.Title,
|
||||
summary: dto.Summary,
|
||||
language: dto.Language,
|
||||
published: dto.Published ?? dto.Modified,
|
||||
modified: dto.Modified ?? dto.Published,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new List<string>(capacity: 4)
|
||||
{
|
||||
dto.SerialNumber,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.SourceId)
|
||||
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
aliases.Add(dto.SourceId);
|
||||
}
|
||||
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>
|
||||
{
|
||||
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
dto.CanonicalUrl,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }))
|
||||
};
|
||||
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
"reference",
|
||||
"cccs",
|
||||
reference.Label,
|
||||
new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
return references
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Products.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Products.Count);
|
||||
for (var index = 0; index < dto.Products.Count; index++)
|
||||
{
|
||||
var product = dto.Products[index];
|
||||
if (string.IsNullOrWhiteSpace(product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var identifier = product.Trim();
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var rangeAnchor = $"cccs:{dto.SerialNumber}:{index}";
|
||||
var versionRanges = BuildVersionRanges(product, rangeAnchor, recordedAt);
|
||||
var normalizedVersions = BuildNormalizedVersions(versionRanges, rangeAnchor);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: versionRanges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: normalizedVersions));
|
||||
}
|
||||
|
||||
return packages.Count == 0
|
||||
? Array.Empty<AffectedPackage>()
|
||||
: packages
|
||||
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(string productText, string rangeAnchor, DateTimeOffset recordedAt)
|
||||
{
|
||||
var versionText = ExtractFirstVersionToken(productText);
|
||||
if (string.IsNullOrWhiteSpace(versionText))
|
||||
{
|
||||
return Array.Empty<AffectedVersionRange>();
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"range",
|
||||
rangeAnchor,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges });
|
||||
|
||||
var vendorExtensions = new Dictionary<string, string>
|
||||
{
|
||||
["cccs.version.raw"] = versionText!,
|
||||
["cccs.anchor"] = rangeAnchor,
|
||||
};
|
||||
|
||||
var semVerResults = SemVerRangeRuleBuilder.Build(versionText!, patchedVersion: null, provenanceNote: rangeAnchor);
|
||||
if (semVerResults.Count > 0)
|
||||
{
|
||||
return semVerResults.Select(result =>
|
||||
new AffectedVersionRange(
|
||||
rangeKind: NormalizedVersionSchemes.SemVer,
|
||||
introducedVersion: result.Primitive.Introduced,
|
||||
fixedVersion: result.Primitive.Fixed,
|
||||
lastAffectedVersion: result.Primitive.LastAffected,
|
||||
rangeExpression: result.Expression ?? versionText!,
|
||||
provenance: provenance,
|
||||
primitives: new RangePrimitives(
|
||||
result.Primitive,
|
||||
Nevra: null,
|
||||
Evr: null,
|
||||
VendorExtensions: vendorExtensions)))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
var primitives = new RangePrimitives(
|
||||
new SemVerPrimitive(
|
||||
Introduced: versionText,
|
||||
IntroducedInclusive: true,
|
||||
Fixed: null,
|
||||
FixedInclusive: false,
|
||||
LastAffected: null,
|
||||
LastAffectedInclusive: true,
|
||||
ConstraintExpression: null,
|
||||
ExactValue: versionText),
|
||||
Nevra: null,
|
||||
Evr: null,
|
||||
VendorExtensions: vendorExtensions);
|
||||
|
||||
return new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: NormalizedVersionSchemes.SemVer,
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: versionText,
|
||||
provenance: provenance,
|
||||
primitives: primitives),
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
IReadOnlyList<AffectedVersionRange> ranges,
|
||||
string rangeAnchor)
|
||||
{
|
||||
if (ranges.Count == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var rules = new List<NormalizedVersionRule>(ranges.Count);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var rule = range.ToNormalizedVersionRule(rangeAnchor);
|
||||
if (rule is not null)
|
||||
{
|
||||
rules.Add(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
|
||||
}
|
||||
|
||||
private static string? ExtractFirstVersionToken(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var match = Regex.Match(value, @"\d+(?:\.\d+){0,3}(?:[A-Za-z0-9\-_]*)?");
|
||||
return match.Success ? match.Value : null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,58 +1,58 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsRawAdvisoryDocument
|
||||
{
|
||||
[JsonPropertyName("sourceId")]
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("serialNumber")]
|
||||
public string? SerialNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("uuid")]
|
||||
public string? Uuid { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "en";
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("canonicalUrl")]
|
||||
public string CanonicalUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("externalUrl")]
|
||||
public string? ExternalUrl { get; init; }
|
||||
|
||||
[JsonPropertyName("bodyHtml")]
|
||||
public string BodyHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("bodySegments")]
|
||||
public string[] BodySegments { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("alertType")]
|
||||
public string? AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("banner")]
|
||||
public string? Banner { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("rawCreated")]
|
||||
public string? RawDateCreated { get; init; }
|
||||
|
||||
[JsonPropertyName("rawModified")]
|
||||
public string? RawDateModified { get; init; }
|
||||
}
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsRawAdvisoryDocument
|
||||
{
|
||||
[JsonPropertyName("sourceId")]
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("serialNumber")]
|
||||
public string? SerialNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("uuid")]
|
||||
public string? Uuid { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "en";
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("canonicalUrl")]
|
||||
public string CanonicalUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("externalUrl")]
|
||||
public string? ExternalUrl { get; init; }
|
||||
|
||||
[JsonPropertyName("bodyHtml")]
|
||||
public string BodyHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("bodySegments")]
|
||||
public string[] BodySegments { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("alertType")]
|
||||
public string? AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("banner")]
|
||||
public string? Banner { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("rawCreated")]
|
||||
public string? RawDateCreated { get; init; }
|
||||
|
||||
[JsonPropertyName("rawModified")]
|
||||
public string? RawDateModified { get; init; }
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
internal static class CccsJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cccs:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CccsFetchJob : IJob
|
||||
{
|
||||
private readonly CccsConnector _connector;
|
||||
|
||||
public CccsFetchJob(CccsConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs;
|
||||
|
||||
internal static class CccsJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cccs:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CccsFetchJob : IJob
|
||||
{
|
||||
private readonly CccsConnector _connector;
|
||||
|
||||
public CccsFetchJob(CccsConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")]
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")]
|
||||
|
||||
@@ -6,7 +6,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Connector.CertBund.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertBund.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
@@ -286,7 +286,7 @@ public sealed class CertBundConnector : IFeedConnector
|
||||
_diagnostics.ParseSuccess(dto.Products.Count, dto.CveIds.Count);
|
||||
parsedCount++;
|
||||
|
||||
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var bson = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", bson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
@@ -428,7 +428,7 @@ public sealed class CertBundConnector : IFeedConnector
|
||||
|
||||
private Task UpdateCursorAsync(CertBundCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var document = cursor.ToDocumentObject();
|
||||
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertBund.Internal;
|
||||
|
||||
@@ -31,13 +31,13 @@ internal sealed record CertBundCursor(
|
||||
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
var document = new DocumentObject
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownAdvisories"] = new BsonArray(KnownAdvisories),
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownAdvisories"] = new DocumentArray(KnownAdvisories),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
@@ -53,7 +53,7 @@ internal sealed record CertBundCursor(
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertBundCursor FromBson(BsonDocument? document)
|
||||
public static CertBundCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
@@ -76,9 +76,9 @@ internal sealed record CertBundCursor(
|
||||
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
|
||||
=> values?.Distinct().ToArray() ?? EmptyGuids;
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
@@ -95,9 +95,9 @@ internal sealed record CertBundCursor(
|
||||
return items;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyStrings;
|
||||
}
|
||||
@@ -108,11 +108,11 @@ internal sealed record CertBundCursor(
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
private static DateTimeOffset? ParseDate(DocumentValue value)
|
||||
=> value.DocumentType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
@@ -338,7 +338,7 @@ public sealed class CertCcConnector : IFeedConnector
|
||||
|
||||
var dto = CertCcNoteParser.Parse(noteBytes, vendorsBytes, vulsBytes, vendorStatusesBytes);
|
||||
var json = JsonSerializer.Serialize(dto, DtoSerializerOptions);
|
||||
var payload = StellaOps.Concelier.Bson.BsonDocument.Parse(json);
|
||||
var payload = StellaOps.Concelier.Documents.DocumentObject.Parse(json);
|
||||
|
||||
_diagnostics.ParseSuccess(
|
||||
dto.Vendors.Count,
|
||||
@@ -678,7 +678,7 @@ public sealed class CertCcConnector : IFeedConnector
|
||||
private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private sealed class NoteDocumentGroup
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
public sealed class CertCcConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-cc";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CertCcConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CertCcConnector>();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
public sealed class CertCcConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-cc";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CertCcConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CertCcConnector>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +1,50 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-cc";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertCcConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertCcFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-cc";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertCcConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertCcFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,37 +1,37 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
public static class CertCcServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertCcOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.BaseApiUri;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(20);
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
|
||||
});
|
||||
|
||||
services.TryAddSingleton<CertCcSummaryPlanner>();
|
||||
services.TryAddSingleton<CertCcDiagnostics>();
|
||||
services.AddTransient<CertCcConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
public static class CertCcServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertCcOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.BaseApiUri;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(20);
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
|
||||
});
|
||||
|
||||
services.TryAddSingleton<CertCcSummaryPlanner>();
|
||||
services.TryAddSingleton<CertCcDiagnostics>();
|
||||
services.AddTransient<CertCcConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,79 +1,79 @@
|
||||
using System;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options governing CERT/CC fetch cadence and API endpoints.
|
||||
/// </summary>
|
||||
public sealed class CertCcOptions
|
||||
{
|
||||
public const string HttpClientName = "certcc";
|
||||
|
||||
/// <summary>
|
||||
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
|
||||
/// </summary>
|
||||
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Sliding window settings controlling which summary endpoints are requested.
|
||||
/// </summary>
|
||||
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
|
||||
{
|
||||
WindowSize = TimeSpan.FromDays(30),
|
||||
Overlap = TimeSpan.FromDays(3),
|
||||
InitialBackfill = TimeSpan.FromDays(365),
|
||||
MinimumWindowSize = TimeSpan.FromDays(1),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of monthly summary endpoints to request in a single plan.
|
||||
/// </summary>
|
||||
public int MaxMonthlySummaries { get; set; } = 6;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
|
||||
/// </summary>
|
||||
public int MaxNotesPerFetch { get; set; } = 25;
|
||||
|
||||
/// <summary>
|
||||
/// Optional delay inserted between successive detail requests to respect upstream throttling.
|
||||
/// </summary>
|
||||
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
|
||||
|
||||
/// <summary>
|
||||
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
|
||||
/// </summary>
|
||||
public bool EnableDetailMapping { get; set; } = true;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
|
||||
}
|
||||
|
||||
SummaryWindow ??= new TimeWindowCursorOptions();
|
||||
SummaryWindow.EnsureValid();
|
||||
|
||||
if (MaxMonthlySummaries <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
|
||||
}
|
||||
|
||||
if (MaxNotesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (DetailRequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options governing CERT/CC fetch cadence and API endpoints.
|
||||
/// </summary>
|
||||
public sealed class CertCcOptions
|
||||
{
|
||||
public const string HttpClientName = "certcc";
|
||||
|
||||
/// <summary>
|
||||
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
|
||||
/// </summary>
|
||||
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Sliding window settings controlling which summary endpoints are requested.
|
||||
/// </summary>
|
||||
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
|
||||
{
|
||||
WindowSize = TimeSpan.FromDays(30),
|
||||
Overlap = TimeSpan.FromDays(3),
|
||||
InitialBackfill = TimeSpan.FromDays(365),
|
||||
MinimumWindowSize = TimeSpan.FromDays(1),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of monthly summary endpoints to request in a single plan.
|
||||
/// </summary>
|
||||
public int MaxMonthlySummaries { get; set; } = 6;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
|
||||
/// </summary>
|
||||
public int MaxNotesPerFetch { get; set; } = 25;
|
||||
|
||||
/// <summary>
|
||||
/// Optional delay inserted between successive detail requests to respect upstream throttling.
|
||||
/// </summary>
|
||||
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
|
||||
|
||||
/// <summary>
|
||||
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
|
||||
/// </summary>
|
||||
public bool EnableDetailMapping { get; set; } = true;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
|
||||
}
|
||||
|
||||
SummaryWindow ??= new TimeWindowCursorOptions();
|
||||
SummaryWindow.EnsureValid();
|
||||
|
||||
if (MaxMonthlySummaries <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
|
||||
}
|
||||
|
||||
if (MaxNotesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (DetailRequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
@@ -22,18 +22,18 @@ internal sealed record CertCcCursor(
|
||||
EmptyGuidArray,
|
||||
null);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new BsonDocument();
|
||||
var document = new DocumentObject();
|
||||
|
||||
var summary = new BsonDocument();
|
||||
var summary = new DocumentObject();
|
||||
SummaryState.WriteTo(summary, "start", "end");
|
||||
document["summary"] = summary;
|
||||
|
||||
document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString()));
|
||||
document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note));
|
||||
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
|
||||
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
|
||||
document["pendingSummaries"] = new DocumentArray(PendingSummaries.Select(static id => id.ToString()));
|
||||
document["pendingNotes"] = new DocumentArray(PendingNotes.Select(static note => note));
|
||||
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString()));
|
||||
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString()));
|
||||
|
||||
if (LastRun.HasValue)
|
||||
{
|
||||
@@ -43,7 +43,7 @@ internal sealed record CertCcCursor(
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertCcCursor FromBson(BsonDocument? document)
|
||||
public static CertCcCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
@@ -51,9 +51,9 @@ internal sealed record CertCcCursor(
|
||||
}
|
||||
|
||||
TimeWindowCursorState summaryState = TimeWindowCursorState.Empty;
|
||||
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument)
|
||||
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is DocumentObject summaryDocument)
|
||||
{
|
||||
summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end");
|
||||
summaryState = TimeWindowCursorState.FromDocumentObject(summaryDocument, "start", "end");
|
||||
}
|
||||
|
||||
var pendingSummaries = ReadGuidArray(document, "pendingSummaries");
|
||||
@@ -64,10 +64,10 @@ internal sealed record CertCcCursor(
|
||||
DateTimeOffset? lastRun = null;
|
||||
if (document.TryGetValue("lastRun", out var lastRunValue))
|
||||
{
|
||||
lastRun = lastRunValue.BsonType switch
|
||||
lastRun = lastRunValue.DocumentType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -93,9 +93,9 @@ internal sealed record CertCcCursor(
|
||||
public CertCcCursor WithLastRun(DateTimeOffset? timestamp)
|
||||
=> this with { LastRun = timestamp };
|
||||
|
||||
private static Guid[] ReadGuidArray(BsonDocument document, string field)
|
||||
private static Guid[] ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyGuidArray;
|
||||
}
|
||||
@@ -112,9 +112,9 @@ internal sealed record CertCcCursor(
|
||||
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
|
||||
}
|
||||
|
||||
private static string[] ReadStringArray(BsonDocument document, string field)
|
||||
private static string[] ReadStringArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyStringArray;
|
||||
}
|
||||
@@ -124,10 +124,10 @@ internal sealed record CertCcCursor(
|
||||
{
|
||||
switch (element)
|
||||
{
|
||||
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
|
||||
case DocumentString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
|
||||
results.Add(bsonString.AsString.Trim());
|
||||
break;
|
||||
case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
|
||||
case DocumentObject bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
|
||||
results.Add(inner.AsString.Trim());
|
||||
break;
|
||||
}
|
||||
@@ -142,14 +142,14 @@ internal sealed record CertCcCursor(
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool TryReadGuid(BsonValue value, out Guid guid)
|
||||
private static bool TryReadGuid(DocumentValue value, out Guid guid)
|
||||
{
|
||||
if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
|
||||
if (value is DocumentString bsonString && Guid.TryParse(bsonString.AsString, out guid))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (value is BsonBinaryData binary)
|
||||
if (value is DocumentBinaryData binary)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
||||
@@ -1,214 +1,214 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
|
||||
/// </summary>
|
||||
public sealed class CertCcDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _planWindows;
|
||||
private readonly Counter<long> _planRequests;
|
||||
private readonly Histogram<double> _planWindowDays;
|
||||
private readonly Counter<long> _summaryFetchAttempts;
|
||||
private readonly Counter<long> _summaryFetchSuccess;
|
||||
private readonly Counter<long> _summaryFetchUnchanged;
|
||||
private readonly Counter<long> _summaryFetchFailures;
|
||||
private readonly Counter<long> _detailFetchAttempts;
|
||||
private readonly Counter<long> _detailFetchSuccess;
|
||||
private readonly Counter<long> _detailFetchUnchanged;
|
||||
private readonly Counter<long> _detailFetchMissing;
|
||||
private readonly Counter<long> _detailFetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseVendorCount;
|
||||
private readonly Histogram<long> _parseStatusCount;
|
||||
private readonly Histogram<long> _parseVulnerabilityCount;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapAffectedPackageCount;
|
||||
private readonly Histogram<long> _mapNormalizedVersionCount;
|
||||
|
||||
public CertCcDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_planWindows = _meter.CreateCounter<long>(
|
||||
name: "certcc.plan.windows",
|
||||
unit: "windows",
|
||||
description: "Number of summary planning windows evaluated.");
|
||||
_planRequests = _meter.CreateCounter<long>(
|
||||
name: "certcc.plan.requests",
|
||||
unit: "requests",
|
||||
description: "Total CERT/CC summary endpoints queued by the planner.");
|
||||
_planWindowDays = _meter.CreateHistogram<double>(
|
||||
name: "certcc.plan.window_days",
|
||||
unit: "day",
|
||||
description: "Duration of each planning window in days.");
|
||||
_summaryFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetch attempts.");
|
||||
_summaryFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches persisted to storage.");
|
||||
_summaryFetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.not_modified",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches returning HTTP 304.");
|
||||
_summaryFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches that failed after retries.");
|
||||
_detailFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetch attempts.");
|
||||
_detailFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches that returned payloads.");
|
||||
_detailFetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.unchanged",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches returning HTTP 304.");
|
||||
_detailFetchMissing = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.missing",
|
||||
unit: "operations",
|
||||
description: "Number of optional VINCE detail endpoints missing but tolerated.");
|
||||
_detailFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches that failed after retries.");
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.parse.success",
|
||||
unit: "documents",
|
||||
description: "Number of VINCE note bundles parsed into DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of VINCE note bundles that failed to parse.");
|
||||
_parseVendorCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.vendors.count",
|
||||
unit: "vendors",
|
||||
description: "Distribution of vendor statements per VINCE note.");
|
||||
_parseStatusCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.statuses.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vendor status entries per VINCE note.");
|
||||
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.vulnerabilities.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vulnerability records per VINCE note.");
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.map.success",
|
||||
unit: "advisories",
|
||||
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
|
||||
_mapFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.map.failures",
|
||||
unit: "advisories",
|
||||
description: "Number of CERT/CC advisory mapping attempts that failed.");
|
||||
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.map.affected.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected packages emitted per CERT/CC advisory.");
|
||||
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.map.normalized_versions.count",
|
||||
unit: "rules",
|
||||
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
|
||||
}
|
||||
|
||||
public void PlanEvaluated(TimeWindow window, int requestCount)
|
||||
{
|
||||
_planWindows.Add(1);
|
||||
|
||||
if (requestCount > 0)
|
||||
{
|
||||
_planRequests.Add(requestCount);
|
||||
}
|
||||
|
||||
var duration = window.Duration;
|
||||
if (duration > TimeSpan.Zero)
|
||||
{
|
||||
_planWindowDays.Record(duration.TotalDays);
|
||||
}
|
||||
}
|
||||
|
||||
public void SummaryFetchAttempt(CertCcSummaryScope scope)
|
||||
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchSuccess(CertCcSummaryScope scope)
|
||||
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
|
||||
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchFailure(CertCcSummaryScope scope)
|
||||
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
|
||||
|
||||
public void DetailFetchAttempt(string endpoint)
|
||||
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchSuccess(string endpoint)
|
||||
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchUnchanged(string endpoint)
|
||||
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchMissing(string endpoint)
|
||||
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchFailure(string endpoint)
|
||||
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
if (vendorCount >= 0)
|
||||
{
|
||||
_parseVendorCount.Record(vendorCount);
|
||||
}
|
||||
if (statusCount >= 0)
|
||||
{
|
||||
_parseStatusCount.Record(statusCount);
|
||||
}
|
||||
if (vulnerabilityCount >= 0)
|
||||
{
|
||||
_parseVulnerabilityCount.Record(vulnerabilityCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseFailure()
|
||||
=> _parseFailures.Add(1);
|
||||
|
||||
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
if (affectedPackageCount >= 0)
|
||||
{
|
||||
_mapAffectedPackageCount.Record(affectedPackageCount);
|
||||
}
|
||||
if (normalizedVersionCount >= 0)
|
||||
{
|
||||
_mapNormalizedVersionCount.Record(normalizedVersionCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void MapFailure()
|
||||
=> _mapFailures.Add(1);
|
||||
|
||||
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
|
||||
=> new("scope", scope.ToString().ToLowerInvariant());
|
||||
|
||||
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
|
||||
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
|
||||
/// </summary>
|
||||
public sealed class CertCcDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _planWindows;
|
||||
private readonly Counter<long> _planRequests;
|
||||
private readonly Histogram<double> _planWindowDays;
|
||||
private readonly Counter<long> _summaryFetchAttempts;
|
||||
private readonly Counter<long> _summaryFetchSuccess;
|
||||
private readonly Counter<long> _summaryFetchUnchanged;
|
||||
private readonly Counter<long> _summaryFetchFailures;
|
||||
private readonly Counter<long> _detailFetchAttempts;
|
||||
private readonly Counter<long> _detailFetchSuccess;
|
||||
private readonly Counter<long> _detailFetchUnchanged;
|
||||
private readonly Counter<long> _detailFetchMissing;
|
||||
private readonly Counter<long> _detailFetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseVendorCount;
|
||||
private readonly Histogram<long> _parseStatusCount;
|
||||
private readonly Histogram<long> _parseVulnerabilityCount;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapAffectedPackageCount;
|
||||
private readonly Histogram<long> _mapNormalizedVersionCount;
|
||||
|
||||
public CertCcDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_planWindows = _meter.CreateCounter<long>(
|
||||
name: "certcc.plan.windows",
|
||||
unit: "windows",
|
||||
description: "Number of summary planning windows evaluated.");
|
||||
_planRequests = _meter.CreateCounter<long>(
|
||||
name: "certcc.plan.requests",
|
||||
unit: "requests",
|
||||
description: "Total CERT/CC summary endpoints queued by the planner.");
|
||||
_planWindowDays = _meter.CreateHistogram<double>(
|
||||
name: "certcc.plan.window_days",
|
||||
unit: "day",
|
||||
description: "Duration of each planning window in days.");
|
||||
_summaryFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetch attempts.");
|
||||
_summaryFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches persisted to storage.");
|
||||
_summaryFetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.not_modified",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches returning HTTP 304.");
|
||||
_summaryFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches that failed after retries.");
|
||||
_detailFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetch attempts.");
|
||||
_detailFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches that returned payloads.");
|
||||
_detailFetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.unchanged",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches returning HTTP 304.");
|
||||
_detailFetchMissing = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.missing",
|
||||
unit: "operations",
|
||||
description: "Number of optional VINCE detail endpoints missing but tolerated.");
|
||||
_detailFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches that failed after retries.");
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.parse.success",
|
||||
unit: "documents",
|
||||
description: "Number of VINCE note bundles parsed into DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of VINCE note bundles that failed to parse.");
|
||||
_parseVendorCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.vendors.count",
|
||||
unit: "vendors",
|
||||
description: "Distribution of vendor statements per VINCE note.");
|
||||
_parseStatusCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.statuses.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vendor status entries per VINCE note.");
|
||||
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.vulnerabilities.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vulnerability records per VINCE note.");
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.map.success",
|
||||
unit: "advisories",
|
||||
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
|
||||
_mapFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.map.failures",
|
||||
unit: "advisories",
|
||||
description: "Number of CERT/CC advisory mapping attempts that failed.");
|
||||
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.map.affected.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected packages emitted per CERT/CC advisory.");
|
||||
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.map.normalized_versions.count",
|
||||
unit: "rules",
|
||||
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
|
||||
}
|
||||
|
||||
public void PlanEvaluated(TimeWindow window, int requestCount)
|
||||
{
|
||||
_planWindows.Add(1);
|
||||
|
||||
if (requestCount > 0)
|
||||
{
|
||||
_planRequests.Add(requestCount);
|
||||
}
|
||||
|
||||
var duration = window.Duration;
|
||||
if (duration > TimeSpan.Zero)
|
||||
{
|
||||
_planWindowDays.Record(duration.TotalDays);
|
||||
}
|
||||
}
|
||||
|
||||
public void SummaryFetchAttempt(CertCcSummaryScope scope)
|
||||
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchSuccess(CertCcSummaryScope scope)
|
||||
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
|
||||
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchFailure(CertCcSummaryScope scope)
|
||||
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
|
||||
|
||||
public void DetailFetchAttempt(string endpoint)
|
||||
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchSuccess(string endpoint)
|
||||
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchUnchanged(string endpoint)
|
||||
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchMissing(string endpoint)
|
||||
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchFailure(string endpoint)
|
||||
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
if (vendorCount >= 0)
|
||||
{
|
||||
_parseVendorCount.Record(vendorCount);
|
||||
}
|
||||
if (statusCount >= 0)
|
||||
{
|
||||
_parseStatusCount.Record(statusCount);
|
||||
}
|
||||
if (vulnerabilityCount >= 0)
|
||||
{
|
||||
_parseVulnerabilityCount.Record(vulnerabilityCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseFailure()
|
||||
=> _parseFailures.Add(1);
|
||||
|
||||
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
if (affectedPackageCount >= 0)
|
||||
{
|
||||
_mapAffectedPackageCount.Record(affectedPackageCount);
|
||||
}
|
||||
if (normalizedVersionCount >= 0)
|
||||
{
|
||||
_mapNormalizedVersionCount.Record(normalizedVersionCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void MapFailure()
|
||||
=> _mapFailures.Add(1);
|
||||
|
||||
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
|
||||
=> new("scope", scope.ToString().ToLowerInvariant());
|
||||
|
||||
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
|
||||
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,97 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal sealed record CertCcNoteDto(
|
||||
CertCcNoteMetadata Metadata,
|
||||
IReadOnlyList<CertCcVendorDto> Vendors,
|
||||
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
|
||||
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
|
||||
{
|
||||
public static CertCcNoteDto Empty { get; } = new(
|
||||
CertCcNoteMetadata.Empty,
|
||||
Array.Empty<CertCcVendorDto>(),
|
||||
Array.Empty<CertCcVendorStatusDto>(),
|
||||
Array.Empty<CertCcVulnerabilityDto>());
|
||||
}
|
||||
|
||||
internal sealed record CertCcNoteMetadata(
|
||||
string? VuId,
|
||||
string IdNumber,
|
||||
string Title,
|
||||
string? Overview,
|
||||
string? Summary,
|
||||
DateTimeOffset? Published,
|
||||
DateTimeOffset? Updated,
|
||||
DateTimeOffset? Created,
|
||||
int? Revision,
|
||||
IReadOnlyList<string> CveIds,
|
||||
IReadOnlyList<string> PublicUrls,
|
||||
string? PrimaryUrl)
|
||||
{
|
||||
public static CertCcNoteMetadata Empty { get; } = new(
|
||||
VuId: null,
|
||||
IdNumber: string.Empty,
|
||||
Title: string.Empty,
|
||||
Overview: null,
|
||||
Summary: null,
|
||||
Published: null,
|
||||
Updated: null,
|
||||
Created: null,
|
||||
Revision: null,
|
||||
CveIds: Array.Empty<string>(),
|
||||
PublicUrls: Array.Empty<string>(),
|
||||
PrimaryUrl: null);
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorDto(
|
||||
string Vendor,
|
||||
DateTimeOffset? ContactDate,
|
||||
DateTimeOffset? StatementDate,
|
||||
DateTimeOffset? Updated,
|
||||
string? Statement,
|
||||
string? Addendum,
|
||||
IReadOnlyList<string> References)
|
||||
{
|
||||
public static CertCcVendorDto Empty { get; } = new(
|
||||
Vendor: string.Empty,
|
||||
ContactDate: null,
|
||||
StatementDate: null,
|
||||
Updated: null,
|
||||
Statement: null,
|
||||
Addendum: null,
|
||||
References: Array.Empty<string>());
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorStatusDto(
|
||||
string Vendor,
|
||||
string CveId,
|
||||
string Status,
|
||||
string? Statement,
|
||||
IReadOnlyList<string> References,
|
||||
DateTimeOffset? DateAdded,
|
||||
DateTimeOffset? DateUpdated)
|
||||
{
|
||||
public static CertCcVendorStatusDto Empty { get; } = new(
|
||||
Vendor: string.Empty,
|
||||
CveId: string.Empty,
|
||||
Status: string.Empty,
|
||||
Statement: null,
|
||||
References: Array.Empty<string>(),
|
||||
DateAdded: null,
|
||||
DateUpdated: null);
|
||||
}
|
||||
|
||||
internal sealed record CertCcVulnerabilityDto(
|
||||
string CveId,
|
||||
string? Description,
|
||||
DateTimeOffset? DateAdded,
|
||||
DateTimeOffset? DateUpdated)
|
||||
{
|
||||
public static CertCcVulnerabilityDto Empty { get; } = new(
|
||||
CveId: string.Empty,
|
||||
Description: null,
|
||||
DateAdded: null,
|
||||
DateUpdated: null);
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal sealed record CertCcNoteDto(
|
||||
CertCcNoteMetadata Metadata,
|
||||
IReadOnlyList<CertCcVendorDto> Vendors,
|
||||
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
|
||||
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
|
||||
{
|
||||
public static CertCcNoteDto Empty { get; } = new(
|
||||
CertCcNoteMetadata.Empty,
|
||||
Array.Empty<CertCcVendorDto>(),
|
||||
Array.Empty<CertCcVendorStatusDto>(),
|
||||
Array.Empty<CertCcVulnerabilityDto>());
|
||||
}
|
||||
|
||||
internal sealed record CertCcNoteMetadata(
|
||||
string? VuId,
|
||||
string IdNumber,
|
||||
string Title,
|
||||
string? Overview,
|
||||
string? Summary,
|
||||
DateTimeOffset? Published,
|
||||
DateTimeOffset? Updated,
|
||||
DateTimeOffset? Created,
|
||||
int? Revision,
|
||||
IReadOnlyList<string> CveIds,
|
||||
IReadOnlyList<string> PublicUrls,
|
||||
string? PrimaryUrl)
|
||||
{
|
||||
public static CertCcNoteMetadata Empty { get; } = new(
|
||||
VuId: null,
|
||||
IdNumber: string.Empty,
|
||||
Title: string.Empty,
|
||||
Overview: null,
|
||||
Summary: null,
|
||||
Published: null,
|
||||
Updated: null,
|
||||
Created: null,
|
||||
Revision: null,
|
||||
CveIds: Array.Empty<string>(),
|
||||
PublicUrls: Array.Empty<string>(),
|
||||
PrimaryUrl: null);
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorDto(
|
||||
string Vendor,
|
||||
DateTimeOffset? ContactDate,
|
||||
DateTimeOffset? StatementDate,
|
||||
DateTimeOffset? Updated,
|
||||
string? Statement,
|
||||
string? Addendum,
|
||||
IReadOnlyList<string> References)
|
||||
{
|
||||
public static CertCcVendorDto Empty { get; } = new(
|
||||
Vendor: string.Empty,
|
||||
ContactDate: null,
|
||||
StatementDate: null,
|
||||
Updated: null,
|
||||
Statement: null,
|
||||
Addendum: null,
|
||||
References: Array.Empty<string>());
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorStatusDto(
|
||||
string Vendor,
|
||||
string CveId,
|
||||
string Status,
|
||||
string? Statement,
|
||||
IReadOnlyList<string> References,
|
||||
DateTimeOffset? DateAdded,
|
||||
DateTimeOffset? DateUpdated)
|
||||
{
|
||||
public static CertCcVendorStatusDto Empty { get; } = new(
|
||||
Vendor: string.Empty,
|
||||
CveId: string.Empty,
|
||||
Status: string.Empty,
|
||||
Statement: null,
|
||||
References: Array.Empty<string>(),
|
||||
DateAdded: null,
|
||||
DateUpdated: null);
|
||||
}
|
||||
|
||||
internal sealed record CertCcVulnerabilityDto(
|
||||
string CveId,
|
||||
string? Description,
|
||||
DateTimeOffset? DateAdded,
|
||||
DateTimeOffset? DateUpdated)
|
||||
{
|
||||
public static CertCcVulnerabilityDto Empty { get; } = new(
|
||||
CveId: string.Empty,
|
||||
Description: null,
|
||||
DateAdded: null,
|
||||
DateUpdated: null);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,108 +1,108 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcSummaryParser
|
||||
{
|
||||
public static IReadOnlyList<string> ParseNotes(byte[] payload)
|
||||
{
|
||||
if (payload is null || payload.Length == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
|
||||
{
|
||||
AllowTrailingCommas = true,
|
||||
CommentHandling = JsonCommentHandling.Skip,
|
||||
});
|
||||
|
||||
var notesElement = document.RootElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
|
||||
JsonValueKind.Array => document.RootElement,
|
||||
JsonValueKind.Null or JsonValueKind.Undefined => default,
|
||||
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
|
||||
};
|
||||
|
||||
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var results = new List<string>(notesElement.GetArrayLength());
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var element in notesElement.EnumerateArray())
|
||||
{
|
||||
var token = ExtractToken(element);
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = token.Trim();
|
||||
var dedupKey = CreateDedupKey(normalized);
|
||||
if (seen.Add(dedupKey))
|
||||
{
|
||||
results.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0 ? Array.Empty<string>() : results;
|
||||
}
|
||||
|
||||
private static string CreateDedupKey(string token)
|
||||
{
|
||||
var digits = string.Concat(token.Where(char.IsDigit));
|
||||
return digits.Length > 0
|
||||
? digits
|
||||
: token.Trim().ToUpperInvariant();
|
||||
}
|
||||
|
||||
private static string? ExtractToken(JsonElement element)
|
||||
{
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => element.GetString(),
|
||||
JsonValueKind.Number => element.TryGetInt64(out var number)
|
||||
? number.ToString(CultureInfo.InvariantCulture)
|
||||
: element.GetRawText(),
|
||||
JsonValueKind.Object => ExtractFromObject(element),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static string? ExtractFromObject(JsonElement element)
|
||||
{
|
||||
foreach (var propertyName in PropertyCandidates)
|
||||
{
|
||||
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var value = property.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static readonly string[] PropertyCandidates =
|
||||
{
|
||||
"note",
|
||||
"notes",
|
||||
"id",
|
||||
"idnumber",
|
||||
"noteId",
|
||||
"vu",
|
||||
"vuid",
|
||||
"vuId",
|
||||
};
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcSummaryParser
|
||||
{
|
||||
public static IReadOnlyList<string> ParseNotes(byte[] payload)
|
||||
{
|
||||
if (payload is null || payload.Length == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
|
||||
{
|
||||
AllowTrailingCommas = true,
|
||||
CommentHandling = JsonCommentHandling.Skip,
|
||||
});
|
||||
|
||||
var notesElement = document.RootElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
|
||||
JsonValueKind.Array => document.RootElement,
|
||||
JsonValueKind.Null or JsonValueKind.Undefined => default,
|
||||
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
|
||||
};
|
||||
|
||||
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var results = new List<string>(notesElement.GetArrayLength());
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var element in notesElement.EnumerateArray())
|
||||
{
|
||||
var token = ExtractToken(element);
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = token.Trim();
|
||||
var dedupKey = CreateDedupKey(normalized);
|
||||
if (seen.Add(dedupKey))
|
||||
{
|
||||
results.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0 ? Array.Empty<string>() : results;
|
||||
}
|
||||
|
||||
private static string CreateDedupKey(string token)
|
||||
{
|
||||
var digits = string.Concat(token.Where(char.IsDigit));
|
||||
return digits.Length > 0
|
||||
? digits
|
||||
: token.Trim().ToUpperInvariant();
|
||||
}
|
||||
|
||||
private static string? ExtractToken(JsonElement element)
|
||||
{
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => element.GetString(),
|
||||
JsonValueKind.Number => element.TryGetInt64(out var number)
|
||||
? number.ToString(CultureInfo.InvariantCulture)
|
||||
: element.GetRawText(),
|
||||
JsonValueKind.Object => ExtractFromObject(element),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static string? ExtractFromObject(JsonElement element)
|
||||
{
|
||||
foreach (var propertyName in PropertyCandidates)
|
||||
{
|
||||
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var value = property.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static readonly string[] PropertyCandidates =
|
||||
{
|
||||
"note",
|
||||
"notes",
|
||||
"id",
|
||||
"idnumber",
|
||||
"noteId",
|
||||
"vu",
|
||||
"vuid",
|
||||
"vuId",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
public sealed record CertCcSummaryPlan(
|
||||
TimeWindow Window,
|
||||
IReadOnlyList<CertCcSummaryRequest> Requests,
|
||||
TimeWindowCursorState NextState);
|
||||
|
||||
public enum CertCcSummaryScope
|
||||
{
|
||||
Monthly,
|
||||
Yearly,
|
||||
}
|
||||
|
||||
public sealed record CertCcSummaryRequest(
|
||||
Uri Uri,
|
||||
CertCcSummaryScope Scope,
|
||||
int Year,
|
||||
int? Month);
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
public sealed record CertCcSummaryPlan(
|
||||
TimeWindow Window,
|
||||
IReadOnlyList<CertCcSummaryRequest> Requests,
|
||||
TimeWindowCursorState NextState);
|
||||
|
||||
public enum CertCcSummaryScope
|
||||
{
|
||||
Monthly,
|
||||
Yearly,
|
||||
}
|
||||
|
||||
public sealed record CertCcSummaryRequest(
|
||||
Uri Uri,
|
||||
CertCcSummaryScope Scope,
|
||||
int Year,
|
||||
int? Month);
|
||||
|
||||
@@ -1,96 +1,96 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
|
||||
/// </summary>
|
||||
public sealed class CertCcSummaryPlanner
|
||||
{
|
||||
private readonly CertCcOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public CertCcSummaryPlanner(
|
||||
IOptions<CertCcOptions> options,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
|
||||
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
|
||||
|
||||
var months = EnumerateYearMonths(window.Start, window.End)
|
||||
.Take(_options.MaxMonthlySummaries)
|
||||
.ToArray();
|
||||
|
||||
if (months.Length == 0)
|
||||
{
|
||||
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
|
||||
}
|
||||
|
||||
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
|
||||
foreach (var month in months)
|
||||
{
|
||||
requests.Add(new CertCcSummaryRequest(
|
||||
BuildMonthlyUri(month.Year, month.Month),
|
||||
CertCcSummaryScope.Monthly,
|
||||
month.Year,
|
||||
month.Month));
|
||||
}
|
||||
|
||||
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
|
||||
{
|
||||
requests.Add(new CertCcSummaryRequest(
|
||||
BuildYearlyUri(year),
|
||||
CertCcSummaryScope.Yearly,
|
||||
year,
|
||||
Month: null));
|
||||
}
|
||||
|
||||
return new CertCcSummaryPlan(window, requests, nextState);
|
||||
}
|
||||
|
||||
private Uri BuildMonthlyUri(int year, int month)
|
||||
{
|
||||
var path = $"{year:D4}/{month:D2}/summary/";
|
||||
return new Uri(_options.BaseApiUri, path);
|
||||
}
|
||||
|
||||
private Uri BuildYearlyUri(int year)
|
||||
{
|
||||
var path = $"{year:D4}/summary/";
|
||||
return new Uri(_options.BaseApiUri, path);
|
||||
}
|
||||
|
||||
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
|
||||
{
|
||||
if (end <= start)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
|
||||
{
|
||||
limit = limit.AddMonths(1);
|
||||
}
|
||||
|
||||
while (cursor < limit)
|
||||
{
|
||||
yield return (cursor.Year, cursor.Month);
|
||||
cursor = cursor.AddMonths(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
|
||||
/// </summary>
|
||||
public sealed class CertCcSummaryPlanner
|
||||
{
|
||||
private readonly CertCcOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public CertCcSummaryPlanner(
|
||||
IOptions<CertCcOptions> options,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
|
||||
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
|
||||
|
||||
var months = EnumerateYearMonths(window.Start, window.End)
|
||||
.Take(_options.MaxMonthlySummaries)
|
||||
.ToArray();
|
||||
|
||||
if (months.Length == 0)
|
||||
{
|
||||
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
|
||||
}
|
||||
|
||||
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
|
||||
foreach (var month in months)
|
||||
{
|
||||
requests.Add(new CertCcSummaryRequest(
|
||||
BuildMonthlyUri(month.Year, month.Month),
|
||||
CertCcSummaryScope.Monthly,
|
||||
month.Year,
|
||||
month.Month));
|
||||
}
|
||||
|
||||
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
|
||||
{
|
||||
requests.Add(new CertCcSummaryRequest(
|
||||
BuildYearlyUri(year),
|
||||
CertCcSummaryScope.Yearly,
|
||||
year,
|
||||
Month: null));
|
||||
}
|
||||
|
||||
return new CertCcSummaryPlan(window, requests, nextState);
|
||||
}
|
||||
|
||||
private Uri BuildMonthlyUri(int year, int month)
|
||||
{
|
||||
var path = $"{year:D4}/{month:D2}/summary/";
|
||||
return new Uri(_options.BaseApiUri, path);
|
||||
}
|
||||
|
||||
private Uri BuildYearlyUri(int year)
|
||||
{
|
||||
var path = $"{year:D4}/summary/";
|
||||
return new Uri(_options.BaseApiUri, path);
|
||||
}
|
||||
|
||||
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
|
||||
{
|
||||
if (end <= start)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
|
||||
{
|
||||
limit = limit.AddMonths(1);
|
||||
}
|
||||
|
||||
while (cursor < limit)
|
||||
{
|
||||
yield return (cursor.Year, cursor.Month);
|
||||
cursor = cursor.AddMonths(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,235 +1,235 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcVendorStatementParser
|
||||
{
|
||||
private static readonly string[] PairSeparators =
|
||||
{
|
||||
"\t",
|
||||
" - ",
|
||||
" – ",
|
||||
" — ",
|
||||
" : ",
|
||||
": ",
|
||||
" :",
|
||||
":",
|
||||
};
|
||||
|
||||
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
|
||||
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
|
||||
|
||||
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
|
||||
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(statement))
|
||||
{
|
||||
return Array.Empty<CertCcVendorPatch>();
|
||||
}
|
||||
|
||||
var patches = new List<CertCcVendorPatch>();
|
||||
var lines = statement
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
foreach (var rawLine in lines)
|
||||
{
|
||||
var line = rawLine.Trim();
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
line = TrimBulletPrefix(line);
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var versions = ExtractVersions(versionSegment);
|
||||
if (versions.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var products = ExtractProducts(productSegment);
|
||||
if (products.Count == 0)
|
||||
{
|
||||
products.Add(string.Empty);
|
||||
}
|
||||
|
||||
if (versions.Count == products.Count)
|
||||
{
|
||||
for (var index = 0; index < versions.Count; index++)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
|
||||
{
|
||||
var groupSize = products.Count / versions.Count;
|
||||
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
|
||||
{
|
||||
var start = versionIndex * groupSize;
|
||||
var end = start + groupSize;
|
||||
var version = versions[versionIndex];
|
||||
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var primaryVersion = versions[0];
|
||||
foreach (var product in products)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
|
||||
}
|
||||
}
|
||||
|
||||
if (patches.Count == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorPatch>();
|
||||
}
|
||||
|
||||
return patches
|
||||
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
|
||||
.Distinct(CertCcVendorPatch.Comparer)
|
||||
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string TrimBulletPrefix(string value)
|
||||
{
|
||||
var trimmed = value.TrimStart(BulletPrefixes).Trim();
|
||||
return trimmed.Length == 0 ? value.Trim() : trimmed;
|
||||
}
|
||||
|
||||
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
|
||||
{
|
||||
foreach (var separator in PairSeparators)
|
||||
{
|
||||
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
productSegment = parts[0];
|
||||
versionSegment = parts[1];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (whitespaceSplit.Length >= 2)
|
||||
{
|
||||
productSegment = string.Join(' ', whitespaceSplit[..^1]);
|
||||
versionSegment = whitespaceSplit[^1];
|
||||
return true;
|
||||
}
|
||||
|
||||
productSegment = string.Empty;
|
||||
versionSegment = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static List<string> ExtractProducts(string segment)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var normalized = segment.Replace('\t', ' ').Trim();
|
||||
var tokens = normalized
|
||||
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(static token => token.Trim())
|
||||
.Where(static token => token.Length > 0)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
private static List<string> ExtractVersions(string segment)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var matches = VersionTokenRegex.Matches(segment);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var versions = new List<string>(matches.Count);
|
||||
foreach (Match match in matches)
|
||||
{
|
||||
if (match.Groups.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = match.Groups[1].Value.Trim();
|
||||
if (value.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
versions.Add(value);
|
||||
}
|
||||
|
||||
return versions
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(32)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
|
||||
{
|
||||
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
|
||||
|
||||
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
|
||||
{
|
||||
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (x is null || y is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode(CertCcVendorPatch obj)
|
||||
{
|
||||
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
|
||||
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
|
||||
return HashCode.Combine(product, version);
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcVendorStatementParser
|
||||
{
|
||||
private static readonly string[] PairSeparators =
|
||||
{
|
||||
"\t",
|
||||
" - ",
|
||||
" – ",
|
||||
" — ",
|
||||
" : ",
|
||||
": ",
|
||||
" :",
|
||||
":",
|
||||
};
|
||||
|
||||
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
|
||||
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
|
||||
|
||||
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
|
||||
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(statement))
|
||||
{
|
||||
return Array.Empty<CertCcVendorPatch>();
|
||||
}
|
||||
|
||||
var patches = new List<CertCcVendorPatch>();
|
||||
var lines = statement
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
foreach (var rawLine in lines)
|
||||
{
|
||||
var line = rawLine.Trim();
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
line = TrimBulletPrefix(line);
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var versions = ExtractVersions(versionSegment);
|
||||
if (versions.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var products = ExtractProducts(productSegment);
|
||||
if (products.Count == 0)
|
||||
{
|
||||
products.Add(string.Empty);
|
||||
}
|
||||
|
||||
if (versions.Count == products.Count)
|
||||
{
|
||||
for (var index = 0; index < versions.Count; index++)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
|
||||
{
|
||||
var groupSize = products.Count / versions.Count;
|
||||
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
|
||||
{
|
||||
var start = versionIndex * groupSize;
|
||||
var end = start + groupSize;
|
||||
var version = versions[versionIndex];
|
||||
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var primaryVersion = versions[0];
|
||||
foreach (var product in products)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
|
||||
}
|
||||
}
|
||||
|
||||
if (patches.Count == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorPatch>();
|
||||
}
|
||||
|
||||
return patches
|
||||
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
|
||||
.Distinct(CertCcVendorPatch.Comparer)
|
||||
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string TrimBulletPrefix(string value)
|
||||
{
|
||||
var trimmed = value.TrimStart(BulletPrefixes).Trim();
|
||||
return trimmed.Length == 0 ? value.Trim() : trimmed;
|
||||
}
|
||||
|
||||
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
|
||||
{
|
||||
foreach (var separator in PairSeparators)
|
||||
{
|
||||
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
productSegment = parts[0];
|
||||
versionSegment = parts[1];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (whitespaceSplit.Length >= 2)
|
||||
{
|
||||
productSegment = string.Join(' ', whitespaceSplit[..^1]);
|
||||
versionSegment = whitespaceSplit[^1];
|
||||
return true;
|
||||
}
|
||||
|
||||
productSegment = string.Empty;
|
||||
versionSegment = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static List<string> ExtractProducts(string segment)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var normalized = segment.Replace('\t', ' ').Trim();
|
||||
var tokens = normalized
|
||||
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(static token => token.Trim())
|
||||
.Where(static token => token.Length > 0)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
private static List<string> ExtractVersions(string segment)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var matches = VersionTokenRegex.Matches(segment);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var versions = new List<string>(matches.Count);
|
||||
foreach (Match match in matches)
|
||||
{
|
||||
if (match.Groups.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = match.Groups[1].Value.Trim();
|
||||
if (value.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
versions.Add(value);
|
||||
}
|
||||
|
||||
return versions
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(32)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
|
||||
{
|
||||
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
|
||||
|
||||
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
|
||||
{
|
||||
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (x is null || y is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode(CertCcVendorPatch obj)
|
||||
{
|
||||
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
|
||||
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
|
||||
return HashCode.Combine(product, version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
internal static class CertCcJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-cc:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CertCcFetchJob : IJob
|
||||
{
|
||||
private readonly CertCcConnector _connector;
|
||||
|
||||
public CertCcFetchJob(CertCcConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc;
|
||||
|
||||
internal static class CertCcJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-cc:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CertCcFetchJob : IJob
|
||||
{
|
||||
private readonly CertCcConnector _connector;
|
||||
|
||||
public CertCcFetchJob(CertCcConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]
|
||||
|
||||
@@ -4,7 +4,7 @@ using System.Linq;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
@@ -236,7 +236,7 @@ public sealed class CertFrConnector : IFeedConnector
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(dto, SerializerOptions);
|
||||
var payload = BsonDocument.Parse(json);
|
||||
var payload = DocumentObject.Parse(json);
|
||||
var validatedAt = _timeProvider.GetUtcNow();
|
||||
|
||||
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
|
||||
@@ -332,6 +332,6 @@ public sealed class CertFrConnector : IFeedConnector
|
||||
private async Task UpdateCursorAsync(CertFrCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
public sealed class CertFrConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-fr";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CertFrConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CertFrConnector>();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
public sealed class CertFrConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-fr";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CertFrConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CertFrConnector>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-fr";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertFrConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertFrFetchJob>();
|
||||
services.AddTransient<CertFrParseJob>();
|
||||
services.AddTransient<CertFrMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob));
|
||||
EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob));
|
||||
EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-fr";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertFrConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertFrFetchJob>();
|
||||
services.AddTransient<CertFrParseJob>();
|
||||
services.AddTransient<CertFrMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob));
|
||||
EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob));
|
||||
EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,36 +1,36 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
public static class CertFrServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action<CertFrOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertFrOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.FeedUri;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0";
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(20);
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
|
||||
});
|
||||
|
||||
services.TryAddSingleton<CertFrFeedClient>();
|
||||
services.AddTransient<CertFrConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
public static class CertFrServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action<CertFrOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertFrOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.FeedUri;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0";
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(20);
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
|
||||
});
|
||||
|
||||
services.TryAddSingleton<CertFrFeedClient>();
|
||||
services.AddTransient<CertFrConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,46 +1,46 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
|
||||
public sealed class CertFrOptions
|
||||
{
|
||||
public const string HttpClientName = "cert-fr";
|
||||
|
||||
public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
|
||||
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
public int MaxItemsPerFetch { get; set; } = 100;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("InitialBackfill must be a positive duration.");
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxItemsPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxItemsPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
|
||||
public sealed class CertFrOptions
|
||||
{
|
||||
public const string HttpClientName = "cert-fr";
|
||||
|
||||
public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
|
||||
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
public int MaxItemsPerFetch { get; set; } = 100;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("InitialBackfill must be a positive duration.");
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxItemsPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxItemsPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,88 +1,88 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal sealed record CertFrCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
public static CertFrCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertFrCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var value)
|
||||
? ParseDate(value)
|
||||
: null;
|
||||
|
||||
return new CertFrCursor(
|
||||
lastPublished,
|
||||
ReadGuidArray(document, "pendingDocuments"),
|
||||
ReadGuidArray(document, "pendingMappings"));
|
||||
}
|
||||
|
||||
public CertFrCursor WithLastPublished(DateTimeOffset? timestamp)
|
||||
=> this with { LastPublished = timestamp };
|
||||
|
||||
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal sealed record CertFrCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
public static CertFrCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
|
||||
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new DocumentObject
|
||||
{
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertFrCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var value)
|
||||
? ParseDate(value)
|
||||
: null;
|
||||
|
||||
return new CertFrCursor(
|
||||
lastPublished,
|
||||
ReadGuidArray(document, "pendingDocuments"),
|
||||
ReadGuidArray(document, "pendingMappings"));
|
||||
}
|
||||
|
||||
public CertFrCursor WithLastPublished(DateTimeOffset? timestamp)
|
||||
=> this with { LastPublished = timestamp };
|
||||
|
||||
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
private static DateTimeOffset? ParseDate(DocumentValue value)
|
||||
=> value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var raw) || raw is not DocumentArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,77 +1,77 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Storage;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal sealed record CertFrDocumentMetadata(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
DateTimeOffset Published,
|
||||
Uri DetailUri,
|
||||
string? Summary)
|
||||
{
|
||||
private const string AdvisoryIdKey = "certfr.advisoryId";
|
||||
private const string TitleKey = "certfr.title";
|
||||
private const string PublishedKey = "certfr.published";
|
||||
private const string SummaryKey = "certfr.summary";
|
||||
|
||||
public static CertFrDocumentMetadata FromDocument(DocumentRecord document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
if (document.Metadata is null)
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR document metadata is missing.");
|
||||
}
|
||||
|
||||
var metadata = document.Metadata;
|
||||
if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR advisory id metadata missing.");
|
||||
}
|
||||
|
||||
if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR title metadata missing.");
|
||||
}
|
||||
|
||||
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR published metadata invalid.");
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR document URI invalid.");
|
||||
}
|
||||
|
||||
metadata.TryGetValue(SummaryKey, out var summary);
|
||||
|
||||
return new CertFrDocumentMetadata(
|
||||
advisoryId.Trim(),
|
||||
title.Trim(),
|
||||
published.ToUniversalTime(),
|
||||
detailUri,
|
||||
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
|
||||
}
|
||||
|
||||
public static IReadOnlyDictionary<string, string> CreateMetadata(CertFrFeedItem item)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(item);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
[AdvisoryIdKey] = item.AdvisoryId,
|
||||
[TitleKey] = item.Title ?? item.AdvisoryId,
|
||||
[PublishedKey] = item.Published.ToString("O"),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Summary))
|
||||
{
|
||||
metadata[SummaryKey] = item.Summary!;
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Storage;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal sealed record CertFrDocumentMetadata(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
DateTimeOffset Published,
|
||||
Uri DetailUri,
|
||||
string? Summary)
|
||||
{
|
||||
private const string AdvisoryIdKey = "certfr.advisoryId";
|
||||
private const string TitleKey = "certfr.title";
|
||||
private const string PublishedKey = "certfr.published";
|
||||
private const string SummaryKey = "certfr.summary";
|
||||
|
||||
public static CertFrDocumentMetadata FromDocument(DocumentRecord document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
if (document.Metadata is null)
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR document metadata is missing.");
|
||||
}
|
||||
|
||||
var metadata = document.Metadata;
|
||||
if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR advisory id metadata missing.");
|
||||
}
|
||||
|
||||
if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR title metadata missing.");
|
||||
}
|
||||
|
||||
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR published metadata invalid.");
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
throw new InvalidOperationException("Cert-FR document URI invalid.");
|
||||
}
|
||||
|
||||
metadata.TryGetValue(SummaryKey, out var summary);
|
||||
|
||||
return new CertFrDocumentMetadata(
|
||||
advisoryId.Trim(),
|
||||
title.Trim(),
|
||||
published.ToUniversalTime(),
|
||||
detailUri,
|
||||
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
|
||||
}
|
||||
|
||||
public static IReadOnlyDictionary<string, string> CreateMetadata(CertFrFeedItem item)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(item);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
[AdvisoryIdKey] = item.AdvisoryId,
|
||||
[TitleKey] = item.Title ?? item.AdvisoryId,
|
||||
[PublishedKey] = item.Published.ToString("O"),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Summary))
|
||||
{
|
||||
metadata[SummaryKey] = item.Summary!;
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal sealed record CertFrDto(
|
||||
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("detailUrl")] string DetailUrl,
|
||||
[property: JsonPropertyName("published")] DateTimeOffset Published,
|
||||
[property: JsonPropertyName("summary")] string? Summary,
|
||||
[property: JsonPropertyName("content")] string Content,
|
||||
[property: JsonPropertyName("references")] IReadOnlyList<string> References);
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal sealed record CertFrDto(
|
||||
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("detailUrl")] string DetailUrl,
|
||||
[property: JsonPropertyName("published")] DateTimeOffset Published,
|
||||
[property: JsonPropertyName("summary")] string? Summary,
|
||||
[property: JsonPropertyName("content")] string Content,
|
||||
[property: JsonPropertyName("references")] IReadOnlyList<string> References);
|
||||
|
||||
@@ -1,109 +1,109 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
public sealed class CertFrFeedClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly CertFrOptions _options;
|
||||
private readonly ILogger<CertFrFeedClient> _logger;
|
||||
|
||||
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CertFrFeedItem>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
|
||||
|
||||
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var document = XDocument.Load(stream);
|
||||
|
||||
var items = new List<CertFrFeedItem>();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
foreach (var itemElement in document.Descendants("item"))
|
||||
{
|
||||
var link = itemElement.Element("link")?.Value;
|
||||
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var title = itemElement.Element("title")?.Value?.Trim();
|
||||
var summary = itemElement.Element("description")?.Value?.Trim();
|
||||
|
||||
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
|
||||
if (published < windowStart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (published > windowEnd)
|
||||
{
|
||||
published = windowEnd;
|
||||
}
|
||||
|
||||
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
|
||||
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
|
||||
}
|
||||
|
||||
return items
|
||||
.OrderBy(item => item.Published)
|
||||
.Take(_options.MaxItemsPerFetch)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParsePublished(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri)
|
||||
{
|
||||
var guid = itemElement.Element("guid")?.Value;
|
||||
if (!string.IsNullOrWhiteSpace(guid))
|
||||
{
|
||||
return guid.Trim();
|
||||
}
|
||||
|
||||
var segments = detailUri.Segments;
|
||||
if (segments.Length > 0)
|
||||
{
|
||||
var slug = segments[^1].Trim('/');
|
||||
if (!string.IsNullOrWhiteSpace(slug))
|
||||
{
|
||||
return slug;
|
||||
}
|
||||
}
|
||||
|
||||
return detailUri.AbsoluteUri;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertFr.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
public sealed class CertFrFeedClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly CertFrOptions _options;
|
||||
private readonly ILogger<CertFrFeedClient> _logger;
|
||||
|
||||
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CertFrFeedItem>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
|
||||
|
||||
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var document = XDocument.Load(stream);
|
||||
|
||||
var items = new List<CertFrFeedItem>();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
foreach (var itemElement in document.Descendants("item"))
|
||||
{
|
||||
var link = itemElement.Element("link")?.Value;
|
||||
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var title = itemElement.Element("title")?.Value?.Trim();
|
||||
var summary = itemElement.Element("description")?.Value?.Trim();
|
||||
|
||||
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
|
||||
if (published < windowStart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (published > windowEnd)
|
||||
{
|
||||
published = windowEnd;
|
||||
}
|
||||
|
||||
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
|
||||
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
|
||||
}
|
||||
|
||||
return items
|
||||
.OrderBy(item => item.Published)
|
||||
.Take(_options.MaxItemsPerFetch)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParsePublished(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri)
|
||||
{
|
||||
var guid = itemElement.Element("guid")?.Value;
|
||||
if (!string.IsNullOrWhiteSpace(guid))
|
||||
{
|
||||
return guid.Trim();
|
||||
}
|
||||
|
||||
var segments = detailUri.Segments;
|
||||
if (segments.Length > 0)
|
||||
{
|
||||
var slug = segments[^1].Trim('/');
|
||||
if (!string.IsNullOrWhiteSpace(slug))
|
||||
{
|
||||
return slug;
|
||||
}
|
||||
}
|
||||
|
||||
return detailUri.AbsoluteUri;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
public sealed record CertFrFeedItem(
|
||||
string AdvisoryId,
|
||||
Uri DetailUri,
|
||||
DateTimeOffset Published,
|
||||
string? Title,
|
||||
string? Summary);
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
public sealed record CertFrFeedItem(
|
||||
string AdvisoryId,
|
||||
Uri DetailUri,
|
||||
DateTimeOffset Published,
|
||||
string? Title,
|
||||
string? Summary);
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal static class CertFrMapper
|
||||
{
|
||||
public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentException.ThrowIfNullOrEmpty(sourceName);
|
||||
|
||||
var advisoryKey = $"cert-fr/{dto.AdvisoryId}";
|
||||
var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime());
|
||||
|
||||
var aliases = new List<string>
|
||||
{
|
||||
$"CERT-FR:{dto.AdvisoryId}",
|
||||
};
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal static class CertFrMapper
|
||||
{
|
||||
public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentException.ThrowIfNullOrEmpty(sourceName);
|
||||
|
||||
var advisoryKey = $"cert-fr/{dto.AdvisoryId}";
|
||||
var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime());
|
||||
|
||||
var aliases = new List<string>
|
||||
{
|
||||
$"CERT-FR:{dto.AdvisoryId}",
|
||||
};
|
||||
|
||||
var references = BuildReferences(dto, provenance).ToArray();
|
||||
|
||||
var affectedPackages = BuildAffectedPackages(dto, provenance).ToArray();
|
||||
@@ -45,22 +45,22 @@ internal static class CertFrMapper
|
||||
var comparer = StringComparer.OrdinalIgnoreCase;
|
||||
var entries = new List<(AdvisoryReference Reference, int Priority)>
|
||||
{
|
||||
(new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0),
|
||||
};
|
||||
|
||||
foreach (var url in dto.References)
|
||||
{
|
||||
entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1));
|
||||
}
|
||||
|
||||
return entries
|
||||
.GroupBy(tuple => tuple.Reference.Url, comparer)
|
||||
.Select(group => group
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.First())
|
||||
.OrderBy(t => t.Priority)
|
||||
(new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0),
|
||||
};
|
||||
|
||||
foreach (var url in dto.References)
|
||||
{
|
||||
entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1));
|
||||
}
|
||||
|
||||
return entries
|
||||
.GroupBy(tuple => tuple.Reference.Url, comparer)
|
||||
.Select(group => group
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.First())
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.Select(t => t.Reference);
|
||||
|
||||
@@ -1,80 +1,80 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal static class CertFrParser
|
||||
{
|
||||
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
|
||||
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
|
||||
|
||||
public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(html);
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
|
||||
var sanitized = SanitizeHtml(html);
|
||||
var summary = BuildSummary(metadata.Summary, sanitized);
|
||||
var references = ExtractReferences(html);
|
||||
|
||||
return new CertFrDto(
|
||||
metadata.AdvisoryId,
|
||||
metadata.Title,
|
||||
metadata.DetailUri.ToString(),
|
||||
metadata.Published,
|
||||
summary,
|
||||
sanitized,
|
||||
references);
|
||||
}
|
||||
|
||||
private static string SanitizeHtml(string html)
|
||||
{
|
||||
var withoutScripts = ScriptRegex.Replace(html, string.Empty);
|
||||
var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty);
|
||||
var withoutTags = TagRegex.Replace(withoutStyles, " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
|
||||
return WhitespaceRegex.Replace(decoded, " ").Trim();
|
||||
}
|
||||
|
||||
private static string? BuildSummary(string? metadataSummary, string content)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(metadataSummary))
|
||||
{
|
||||
return metadataSummary.Trim();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (sentences.Length > 0)
|
||||
{
|
||||
return sentences[0].Trim();
|
||||
}
|
||||
|
||||
return content.Length > 280 ? content[..280].Trim() : content;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractReferences(string html)
|
||||
{
|
||||
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in AnchorRegex.Matches(html))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
references.Add(match.Groups["url"].Value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr.Internal;
|
||||
|
||||
internal static class CertFrParser
|
||||
{
|
||||
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
|
||||
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
|
||||
|
||||
public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(html);
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
|
||||
var sanitized = SanitizeHtml(html);
|
||||
var summary = BuildSummary(metadata.Summary, sanitized);
|
||||
var references = ExtractReferences(html);
|
||||
|
||||
return new CertFrDto(
|
||||
metadata.AdvisoryId,
|
||||
metadata.Title,
|
||||
metadata.DetailUri.ToString(),
|
||||
metadata.Published,
|
||||
summary,
|
||||
sanitized,
|
||||
references);
|
||||
}
|
||||
|
||||
private static string SanitizeHtml(string html)
|
||||
{
|
||||
var withoutScripts = ScriptRegex.Replace(html, string.Empty);
|
||||
var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty);
|
||||
var withoutTags = TagRegex.Replace(withoutStyles, " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
|
||||
return WhitespaceRegex.Replace(decoded, " ").Trim();
|
||||
}
|
||||
|
||||
private static string? BuildSummary(string? metadataSummary, string content)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(metadataSummary))
|
||||
{
|
||||
return metadataSummary.Trim();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (sentences.Length > 0)
|
||||
{
|
||||
return sentences[0].Trim();
|
||||
}
|
||||
|
||||
return content.Length > 280 ? content[..280].Trim() : content;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractReferences(string html)
|
||||
{
|
||||
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in AnchorRegex.Matches(html))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
references.Add(match.Groups["url"].Value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,46 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
internal static class CertFrJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-fr:fetch";
|
||||
public const string Parse = "source:cert-fr:parse";
|
||||
public const string Map = "source:cert-fr:map";
|
||||
}
|
||||
|
||||
internal sealed class CertFrFetchJob : IJob
|
||||
{
|
||||
private readonly CertFrConnector _connector;
|
||||
|
||||
public CertFrFetchJob(CertFrConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertFrParseJob : IJob
|
||||
{
|
||||
private readonly CertFrConnector _connector;
|
||||
|
||||
public CertFrParseJob(CertFrConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertFrMapJob : IJob
|
||||
{
|
||||
private readonly CertFrConnector _connector;
|
||||
|
||||
public CertFrMapJob(CertFrConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertFr;
|
||||
|
||||
internal static class CertFrJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-fr:fetch";
|
||||
public const string Parse = "source:cert-fr:parse";
|
||||
public const string Map = "source:cert-fr:map";
|
||||
}
|
||||
|
||||
internal sealed class CertFrFetchJob : IJob
|
||||
{
|
||||
private readonly CertFrConnector _connector;
|
||||
|
||||
public CertFrFetchJob(CertFrConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertFrParseJob : IJob
|
||||
{
|
||||
private readonly CertFrConnector _connector;
|
||||
|
||||
public CertFrParseJob(CertFrConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertFrMapJob : IJob
|
||||
{
|
||||
private readonly CertFrConnector _connector;
|
||||
|
||||
public CertFrMapJob(CertFrConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
@@ -226,7 +226,7 @@ public sealed class CertInConnector : IFeedConnector
|
||||
}
|
||||
|
||||
var dto = CertInDetailParser.Parse(listing, rawBytes);
|
||||
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var payload = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certin.v1", payload, _timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
@@ -271,9 +271,9 @@ public sealed class CertInConnector : IFeedConnector
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Bson.IO.JsonWriterSettings
|
||||
var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Documents.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = StellaOps.Concelier.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
OutputMode = StellaOps.Concelier.Documents.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
CertInAdvisoryDto dto;
|
||||
@@ -423,7 +423,7 @@ public sealed class CertInConnector : IFeedConnector
|
||||
|
||||
private Task UpdateCursorAsync(CertInCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken);
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), _timeProvider.GetUtcNow(), cancellationToken);
|
||||
}
|
||||
|
||||
private static bool TryDeserializeListing(IReadOnlyDictionary<string, string>? metadata, out CertInListingItem listing)
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
public sealed class CertInConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-in";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<CertInConnector>(services);
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
public sealed class CertInConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-in";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<CertInConnector>(services);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-in";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertInConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertInFetchJob>();
|
||||
services.AddTransient<CertInParseJob>();
|
||||
services.AddTransient<CertInMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob));
|
||||
EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob));
|
||||
EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-in";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertInConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertInFetchJob>();
|
||||
services.AddTransient<CertInParseJob>();
|
||||
services.AddTransient<CertInMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob));
|
||||
EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob));
|
||||
EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,37 +1,37 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
public static class CertInServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action<CertInOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertInOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.AlertsEndpoint;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
});
|
||||
|
||||
services.AddTransient<CertInClient>();
|
||||
services.AddTransient<CertInConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
public static class CertInServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action<CertInOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertInOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.AlertsEndpoint;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
});
|
||||
|
||||
services.AddTransient<CertInClient>();
|
||||
services.AddTransient<CertInConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,68 +1,68 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
|
||||
public sealed class CertInOptions
|
||||
{
|
||||
public static string HttpClientName => "source.certin";
|
||||
|
||||
/// <summary>
|
||||
/// Endpoint returning a paginated list of recent advisories.
|
||||
/// </summary>
|
||||
public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Size of the rolling fetch window.
|
||||
/// </summary>
|
||||
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
/// <summary>
|
||||
/// Overlap applied to subsequent windows.
|
||||
/// </summary>
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum pages fetched per cycle.
|
||||
/// </summary>
|
||||
public int MaxPagesPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between successive HTTP requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
|
||||
|
||||
[MemberNotNull(nameof(AlertsEndpoint))]
|
||||
public void Validate()
|
||||
{
|
||||
if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("AlertsEndpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (WindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowSize must be greater than zero.");
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (WindowOverlap >= WindowSize)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
|
||||
}
|
||||
|
||||
if (MaxPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
|
||||
public sealed class CertInOptions
|
||||
{
|
||||
public static string HttpClientName => "source.certin";
|
||||
|
||||
/// <summary>
|
||||
/// Endpoint returning a paginated list of recent advisories.
|
||||
/// </summary>
|
||||
public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Size of the rolling fetch window.
|
||||
/// </summary>
|
||||
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
/// <summary>
|
||||
/// Overlap applied to subsequent windows.
|
||||
/// </summary>
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum pages fetched per cycle.
|
||||
/// </summary>
|
||||
public int MaxPagesPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between successive HTTP requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
|
||||
|
||||
[MemberNotNull(nameof(AlertsEndpoint))]
|
||||
public void Validate()
|
||||
{
|
||||
if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("AlertsEndpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (WindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowSize must be greater than zero.");
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (WindowOverlap >= WindowSize)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
|
||||
}
|
||||
|
||||
if (MaxPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
internal sealed record CertInAdvisoryDto(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
string Link,
|
||||
DateTimeOffset Published,
|
||||
string? Summary,
|
||||
string Content,
|
||||
string? Severity,
|
||||
ImmutableArray<string> CveIds,
|
||||
ImmutableArray<string> VendorNames,
|
||||
ImmutableArray<string> ReferenceLinks);
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
internal sealed record CertInAdvisoryDto(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
string Link,
|
||||
DateTimeOffset Published,
|
||||
string? Summary,
|
||||
string Content,
|
||||
string? Severity,
|
||||
ImmutableArray<string> CveIds,
|
||||
ImmutableArray<string> VendorNames,
|
||||
ImmutableArray<string> ReferenceLinks);
|
||||
|
||||
@@ -1,129 +1,129 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
public sealed class CertInClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly CertInOptions _options;
|
||||
private readonly ILogger<CertInClient> _logger;
|
||||
|
||||
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
|
||||
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
|
||||
|
||||
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var root = document.RootElement;
|
||||
if (root.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
|
||||
return Array.Empty<CertInListingItem>();
|
||||
}
|
||||
|
||||
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
|
||||
foreach (var element in root.EnumerateArray())
|
||||
{
|
||||
if (!TryParseListing(element, out var item))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
|
||||
{
|
||||
item = null!;
|
||||
|
||||
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var advisoryId = idElement.GetString();
|
||||
if (string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
|
||||
? titleElement.GetString()
|
||||
: advisoryId;
|
||||
|
||||
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
DateTimeOffset published;
|
||||
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string? summary = null;
|
||||
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
summary = summaryElement.GetString();
|
||||
}
|
||||
|
||||
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Uri BuildPageUri(Uri baseUri, int page)
|
||||
{
|
||||
if (page <= 1)
|
||||
{
|
||||
return baseUri;
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(baseUri);
|
||||
var trimmed = builder.Query.TrimStart('?');
|
||||
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
|
||||
builder.Query = string.IsNullOrEmpty(trimmed)
|
||||
? pageSegment
|
||||
: $"{trimmed}&{pageSegment}";
|
||||
return builder.Uri;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
public sealed class CertInClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly CertInOptions _options;
|
||||
private readonly ILogger<CertInClient> _logger;
|
||||
|
||||
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
|
||||
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
|
||||
|
||||
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var root = document.RootElement;
|
||||
if (root.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
|
||||
return Array.Empty<CertInListingItem>();
|
||||
}
|
||||
|
||||
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
|
||||
foreach (var element in root.EnumerateArray())
|
||||
{
|
||||
if (!TryParseListing(element, out var item))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
|
||||
{
|
||||
item = null!;
|
||||
|
||||
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var advisoryId = idElement.GetString();
|
||||
if (string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
|
||||
? titleElement.GetString()
|
||||
: advisoryId;
|
||||
|
||||
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
DateTimeOffset published;
|
||||
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string? summary = null;
|
||||
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
summary = summaryElement.GetString();
|
||||
}
|
||||
|
||||
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Uri BuildPageUri(Uri baseUri, int page)
|
||||
{
|
||||
if (page <= 1)
|
||||
{
|
||||
return baseUri;
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(baseUri);
|
||||
var trimmed = builder.Query.TrimStart('?');
|
||||
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
|
||||
builder.Query = string.IsNullOrEmpty(trimmed)
|
||||
? pageSegment
|
||||
: $"{trimmed}&{pageSegment}";
|
||||
return builder.Uri;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,88 +1,88 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
internal sealed record CertInCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertInCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
|
||||
? ParseDate(dateValue)
|
||||
: null;
|
||||
|
||||
return new CertInCursor(
|
||||
lastPublished,
|
||||
ReadGuidArray(document, "pendingDocuments"),
|
||||
ReadGuidArray(document, "pendingMappings"));
|
||||
}
|
||||
|
||||
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
|
||||
=> this with { LastPublished = timestamp };
|
||||
|
||||
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
internal sealed record CertInCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
|
||||
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new DocumentObject
|
||||
{
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertInCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
|
||||
? ParseDate(dateValue)
|
||||
: null;
|
||||
|
||||
return new CertInCursor(
|
||||
lastPublished,
|
||||
ReadGuidArray(document, "pendingDocuments"),
|
||||
ReadGuidArray(document, "pendingMappings"));
|
||||
}
|
||||
|
||||
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
|
||||
=> this with { LastPublished = timestamp };
|
||||
|
||||
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
private static DateTimeOffset? ParseDate(DocumentValue value)
|
||||
=> value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,187 +1,187 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
internal static class CertInDetailParser
|
||||
{
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(listing);
|
||||
|
||||
var html = Encoding.UTF8.GetString(rawHtml);
|
||||
var content = HtmlToPlainText(html);
|
||||
var summary = listing.Summary ?? ExtractSummary(content);
|
||||
var severity = ExtractSeverity(content);
|
||||
var cves = ExtractCves(listing.Title, summary, content);
|
||||
var vendors = ExtractVendors(summary, content);
|
||||
var references = ExtractLinks(html);
|
||||
|
||||
return new CertInAdvisoryDto(
|
||||
listing.AdvisoryId,
|
||||
listing.Title,
|
||||
listing.DetailUri.ToString(),
|
||||
listing.Published,
|
||||
summary,
|
||||
content,
|
||||
severity,
|
||||
cves,
|
||||
vendors,
|
||||
references);
|
||||
}
|
||||
|
||||
private static string HtmlToPlainText(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
|
||||
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
|
||||
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
|
||||
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
|
||||
return string.IsNullOrWhiteSpace(decoded)
|
||||
? string.Empty
|
||||
: Regex.Replace(decoded, "\\s+", " ").Trim();
|
||||
}
|
||||
|
||||
private static string? ExtractSummary(string content)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sentenceTerminators = new[] { ".", "!", "?" };
|
||||
foreach (var terminator in sentenceTerminators)
|
||||
{
|
||||
var index = content.IndexOf(terminator, StringComparison.Ordinal);
|
||||
if (index > 0)
|
||||
{
|
||||
return content[..(index + terminator.Length)].Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return content.Length > 280 ? content[..280].Trim() : content;
|
||||
}
|
||||
|
||||
private static string? ExtractSeverity(string content)
|
||||
{
|
||||
var match = SeverityRegex.Match(content);
|
||||
if (match.Success)
|
||||
{
|
||||
return match.Groups["value"].Value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void Capture(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (Match match in CveRegex.Matches(text))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
set.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Capture(title);
|
||||
Capture(summary);
|
||||
Capture(content);
|
||||
|
||||
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
|
||||
{
|
||||
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void Add(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var cleaned = value
|
||||
.Replace("’", "'", StringComparison.Ordinal)
|
||||
.Trim();
|
||||
|
||||
if (cleaned.Length > 200)
|
||||
{
|
||||
cleaned = cleaned[..200];
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cleaned))
|
||||
{
|
||||
vendors.Add(cleaned);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(summary))
|
||||
{
|
||||
foreach (Match match in VendorRegex.Matches(summary))
|
||||
{
|
||||
Add(match.Groups["value"].Value);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Match match in VendorRegex.Matches(content))
|
||||
{
|
||||
Add(match.Groups["value"].Value);
|
||||
}
|
||||
|
||||
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
|
||||
{
|
||||
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
|
||||
Add(fallback);
|
||||
}
|
||||
|
||||
return vendors.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractLinks(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in LinkRegex.Matches(html))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
links.Add(match.Groups[1].Value);
|
||||
}
|
||||
}
|
||||
|
||||
return links.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
internal static class CertInDetailParser
|
||||
{
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(listing);
|
||||
|
||||
var html = Encoding.UTF8.GetString(rawHtml);
|
||||
var content = HtmlToPlainText(html);
|
||||
var summary = listing.Summary ?? ExtractSummary(content);
|
||||
var severity = ExtractSeverity(content);
|
||||
var cves = ExtractCves(listing.Title, summary, content);
|
||||
var vendors = ExtractVendors(summary, content);
|
||||
var references = ExtractLinks(html);
|
||||
|
||||
return new CertInAdvisoryDto(
|
||||
listing.AdvisoryId,
|
||||
listing.Title,
|
||||
listing.DetailUri.ToString(),
|
||||
listing.Published,
|
||||
summary,
|
||||
content,
|
||||
severity,
|
||||
cves,
|
||||
vendors,
|
||||
references);
|
||||
}
|
||||
|
||||
private static string HtmlToPlainText(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
|
||||
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
|
||||
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
|
||||
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
|
||||
return string.IsNullOrWhiteSpace(decoded)
|
||||
? string.Empty
|
||||
: Regex.Replace(decoded, "\\s+", " ").Trim();
|
||||
}
|
||||
|
||||
private static string? ExtractSummary(string content)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sentenceTerminators = new[] { ".", "!", "?" };
|
||||
foreach (var terminator in sentenceTerminators)
|
||||
{
|
||||
var index = content.IndexOf(terminator, StringComparison.Ordinal);
|
||||
if (index > 0)
|
||||
{
|
||||
return content[..(index + terminator.Length)].Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return content.Length > 280 ? content[..280].Trim() : content;
|
||||
}
|
||||
|
||||
private static string? ExtractSeverity(string content)
|
||||
{
|
||||
var match = SeverityRegex.Match(content);
|
||||
if (match.Success)
|
||||
{
|
||||
return match.Groups["value"].Value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void Capture(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (Match match in CveRegex.Matches(text))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
set.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Capture(title);
|
||||
Capture(summary);
|
||||
Capture(content);
|
||||
|
||||
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
|
||||
{
|
||||
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void Add(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var cleaned = value
|
||||
.Replace("’", "'", StringComparison.Ordinal)
|
||||
.Trim();
|
||||
|
||||
if (cleaned.Length > 200)
|
||||
{
|
||||
cleaned = cleaned[..200];
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cleaned))
|
||||
{
|
||||
vendors.Add(cleaned);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(summary))
|
||||
{
|
||||
foreach (Match match in VendorRegex.Matches(summary))
|
||||
{
|
||||
Add(match.Groups["value"].Value);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Match match in VendorRegex.Matches(content))
|
||||
{
|
||||
Add(match.Groups["value"].Value);
|
||||
}
|
||||
|
||||
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
|
||||
{
|
||||
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
|
||||
Add(fallback);
|
||||
}
|
||||
|
||||
return vendors.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractLinks(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in LinkRegex.Matches(html))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
links.Add(match.Groups[1].Value);
|
||||
}
|
||||
}
|
||||
|
||||
return links.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
public sealed record CertInListingItem(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
Uri DetailUri,
|
||||
DateTimeOffset Published,
|
||||
string? Summary);
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
|
||||
public sealed record CertInListingItem(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
Uri DetailUri,
|
||||
DateTimeOffset Published,
|
||||
string? Summary);
|
||||
|
||||
@@ -1,46 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
internal static class CertInJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-in:fetch";
|
||||
public const string Parse = "source:cert-in:parse";
|
||||
public const string Map = "source:cert-in:map";
|
||||
}
|
||||
|
||||
internal sealed class CertInFetchJob : IJob
|
||||
{
|
||||
private readonly CertInConnector _connector;
|
||||
|
||||
public CertInFetchJob(CertInConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertInParseJob : IJob
|
||||
{
|
||||
private readonly CertInConnector _connector;
|
||||
|
||||
public CertInParseJob(CertInConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertInMapJob : IJob
|
||||
{
|
||||
private readonly CertInConnector _connector;
|
||||
|
||||
public CertInMapJob(CertInConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn;
|
||||
|
||||
internal static class CertInJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-in:fetch";
|
||||
public const string Parse = "source:cert-in:parse";
|
||||
public const string Map = "source:cert-in:map";
|
||||
}
|
||||
|
||||
internal sealed class CertInFetchJob : IJob
|
||||
{
|
||||
private readonly CertInConnector _connector;
|
||||
|
||||
public CertInFetchJob(CertInConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertInParseJob : IJob
|
||||
{
|
||||
private readonly CertInConnector _connector;
|
||||
|
||||
public CertInParseJob(CertInConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CertInMapJob : IJob
|
||||
{
|
||||
private readonly CertInConnector _connector;
|
||||
|
||||
public CertInMapJob(CertInConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,29 +1,29 @@
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
|
||||
/// </summary>
|
||||
public static class PaginationPlanner
|
||||
{
|
||||
/// <summary>
|
||||
/// Enumerates additional page start indices given the total result count returned by the source.
|
||||
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
|
||||
/// </summary>
|
||||
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
|
||||
{
|
||||
if (totalResults <= 0 || resultsPerPage <= 0)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (firstPageStartIndex < 0)
|
||||
{
|
||||
firstPageStartIndex = 0;
|
||||
}
|
||||
|
||||
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
|
||||
{
|
||||
yield return start;
|
||||
}
|
||||
}
|
||||
}
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
|
||||
/// </summary>
|
||||
public static class PaginationPlanner
|
||||
{
|
||||
/// <summary>
|
||||
/// Enumerates additional page start indices given the total result count returned by the source.
|
||||
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
|
||||
/// </summary>
|
||||
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
|
||||
{
|
||||
if (totalResults <= 0 || resultsPerPage <= 0)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (firstPageStartIndex < 0)
|
||||
{
|
||||
firstPageStartIndex = 0;
|
||||
}
|
||||
|
||||
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
|
||||
{
|
||||
yield return start;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,43 +1,43 @@
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration applied when advancing sliding time-window cursors.
|
||||
/// </summary>
|
||||
public sealed class TimeWindowCursorOptions
|
||||
{
|
||||
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
|
||||
|
||||
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
|
||||
|
||||
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public void EnsureValid()
|
||||
{
|
||||
if (WindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Window size must be positive.");
|
||||
}
|
||||
|
||||
if (Overlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Window overlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (Overlap >= WindowSize)
|
||||
{
|
||||
throw new InvalidOperationException("Window overlap must be less than the window size.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Initial backfill must be positive.");
|
||||
}
|
||||
|
||||
if (MinimumWindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Minimum window size must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration applied when advancing sliding time-window cursors.
|
||||
/// </summary>
|
||||
public sealed class TimeWindowCursorOptions
|
||||
{
|
||||
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
|
||||
|
||||
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
|
||||
|
||||
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public void EnsureValid()
|
||||
{
|
||||
if (WindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Window size must be positive.");
|
||||
}
|
||||
|
||||
if (Overlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Window overlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (Overlap >= WindowSize)
|
||||
{
|
||||
throw new InvalidOperationException("Window overlap must be less than the window size.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Initial backfill must be positive.");
|
||||
}
|
||||
|
||||
if (MinimumWindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Minimum window size must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +1,50 @@
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Utility methods for computing sliding time-window ranges used by connectors.
|
||||
/// </summary>
|
||||
public static class TimeWindowCursorPlanner
|
||||
{
|
||||
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
options.EnsureValid();
|
||||
|
||||
var effectiveState = state ?? TimeWindowCursorState.Empty;
|
||||
|
||||
var earliest = now - options.InitialBackfill;
|
||||
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
|
||||
if (anchorEnd < earliest)
|
||||
{
|
||||
anchorEnd = earliest;
|
||||
}
|
||||
|
||||
var start = anchorEnd - options.Overlap;
|
||||
if (start < earliest)
|
||||
{
|
||||
start = earliest;
|
||||
}
|
||||
|
||||
var end = start + options.WindowSize;
|
||||
if (end > now)
|
||||
{
|
||||
end = now;
|
||||
}
|
||||
|
||||
if (end <= start)
|
||||
{
|
||||
end = start + options.MinimumWindowSize;
|
||||
if (end > now)
|
||||
{
|
||||
end = now;
|
||||
}
|
||||
}
|
||||
|
||||
if (end <= start)
|
||||
{
|
||||
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
|
||||
}
|
||||
|
||||
return new TimeWindow(start, end);
|
||||
}
|
||||
}
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Utility methods for computing sliding time-window ranges used by connectors.
|
||||
/// </summary>
|
||||
public static class TimeWindowCursorPlanner
|
||||
{
|
||||
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
options.EnsureValid();
|
||||
|
||||
var effectiveState = state ?? TimeWindowCursorState.Empty;
|
||||
|
||||
var earliest = now - options.InitialBackfill;
|
||||
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
|
||||
if (anchorEnd < earliest)
|
||||
{
|
||||
anchorEnd = earliest;
|
||||
}
|
||||
|
||||
var start = anchorEnd - options.Overlap;
|
||||
if (start < earliest)
|
||||
{
|
||||
start = earliest;
|
||||
}
|
||||
|
||||
var end = start + options.WindowSize;
|
||||
if (end > now)
|
||||
{
|
||||
end = now;
|
||||
}
|
||||
|
||||
if (end <= start)
|
||||
{
|
||||
end = start + options.MinimumWindowSize;
|
||||
if (end > now)
|
||||
{
|
||||
end = now;
|
||||
}
|
||||
}
|
||||
|
||||
if (end <= start)
|
||||
{
|
||||
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
|
||||
}
|
||||
|
||||
return new TimeWindow(start, end);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,84 +1,84 @@
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the persisted state of a sliding time-window cursor.
|
||||
/// </summary>
|
||||
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
|
||||
{
|
||||
public static TimeWindowCursorState Empty { get; } = new(null, null);
|
||||
|
||||
public TimeWindowCursorState WithWindow(TimeWindow window)
|
||||
{
|
||||
return new TimeWindowCursorState(window.Start, window.End);
|
||||
}
|
||||
|
||||
public BsonDocument ToBsonDocument(string startField = "windowStart", string endField = "windowEnd")
|
||||
{
|
||||
var document = new BsonDocument();
|
||||
WriteTo(document, startField, endField);
|
||||
return document;
|
||||
}
|
||||
|
||||
public void WriteTo(BsonDocument document, string startField = "windowStart", string endField = "windowEnd")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentException.ThrowIfNullOrEmpty(startField);
|
||||
ArgumentException.ThrowIfNullOrEmpty(endField);
|
||||
|
||||
document.Remove(startField);
|
||||
document.Remove(endField);
|
||||
|
||||
if (LastWindowStart.HasValue)
|
||||
{
|
||||
document[startField] = LastWindowStart.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (LastWindowEnd.HasValue)
|
||||
{
|
||||
document[endField] = LastWindowEnd.Value.UtcDateTime;
|
||||
}
|
||||
}
|
||||
|
||||
public static TimeWindowCursorState FromBsonDocument(BsonDocument? document, string startField = "windowStart", string endField = "windowEnd")
|
||||
{
|
||||
if (document is null)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
DateTimeOffset? start = null;
|
||||
DateTimeOffset? end = null;
|
||||
|
||||
if (document.TryGetValue(startField, out var startValue))
|
||||
{
|
||||
start = ReadDateTimeOffset(startValue);
|
||||
}
|
||||
|
||||
if (document.TryGetValue(endField, out var endValue))
|
||||
{
|
||||
end = ReadDateTimeOffset(endValue);
|
||||
}
|
||||
|
||||
return new TimeWindowCursorState(start, end);
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ReadDateTimeOffset(BsonValue value)
|
||||
{
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple value object describing a time window.
|
||||
/// </summary>
|
||||
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
|
||||
{
|
||||
public TimeSpan Duration => End - Start;
|
||||
}
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the persisted state of a sliding time-window cursor.
|
||||
/// </summary>
|
||||
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
|
||||
{
|
||||
public static TimeWindowCursorState Empty { get; } = new(null, null);
|
||||
|
||||
public TimeWindowCursorState WithWindow(TimeWindow window)
|
||||
{
|
||||
return new TimeWindowCursorState(window.Start, window.End);
|
||||
}
|
||||
|
||||
public DocumentObject ToDocumentObject(string startField = "windowStart", string endField = "windowEnd")
|
||||
{
|
||||
var document = new DocumentObject();
|
||||
WriteTo(document, startField, endField);
|
||||
return document;
|
||||
}
|
||||
|
||||
public void WriteTo(DocumentObject document, string startField = "windowStart", string endField = "windowEnd")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentException.ThrowIfNullOrEmpty(startField);
|
||||
ArgumentException.ThrowIfNullOrEmpty(endField);
|
||||
|
||||
document.Remove(startField);
|
||||
document.Remove(endField);
|
||||
|
||||
if (LastWindowStart.HasValue)
|
||||
{
|
||||
document[startField] = LastWindowStart.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (LastWindowEnd.HasValue)
|
||||
{
|
||||
document[endField] = LastWindowEnd.Value.UtcDateTime;
|
||||
}
|
||||
}
|
||||
|
||||
public static TimeWindowCursorState FromDocumentObject(DocumentObject? document, string startField = "windowStart", string endField = "windowEnd")
|
||||
{
|
||||
if (document is null)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
DateTimeOffset? start = null;
|
||||
DateTimeOffset? end = null;
|
||||
|
||||
if (document.TryGetValue(startField, out var startValue))
|
||||
{
|
||||
start = ReadDateTimeOffset(startValue);
|
||||
}
|
||||
|
||||
if (document.TryGetValue(endField, out var endValue))
|
||||
{
|
||||
end = ReadDateTimeOffset(endValue);
|
||||
}
|
||||
|
||||
return new TimeWindowCursorState(start, end);
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ReadDateTimeOffset(DocumentValue value)
|
||||
{
|
||||
return value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple value object describing a time window.
|
||||
/// </summary>
|
||||
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
|
||||
{
|
||||
public TimeSpan Duration => End - Start;
|
||||
}
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
namespace StellaOps.Concelier.Connector.Common;
|
||||
|
||||
/// <summary>
|
||||
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
|
||||
/// </summary>
|
||||
public static class DocumentStatuses
|
||||
{
|
||||
/// <summary>
|
||||
/// Document captured from the upstream source and awaiting schema validation/parsing.
|
||||
/// </summary>
|
||||
public const string PendingParse = "pending-parse";
|
||||
|
||||
/// <summary>
|
||||
/// Document parsed and sanitized; awaiting canonical mapping.
|
||||
/// </summary>
|
||||
public const string PendingMap = "pending-map";
|
||||
|
||||
/// <summary>
|
||||
/// Document fully mapped to canonical advisories.
|
||||
/// </summary>
|
||||
public const string Mapped = "mapped";
|
||||
|
||||
/// <summary>
|
||||
/// Document failed processing; requires manual intervention before retry.
|
||||
/// </summary>
|
||||
public const string Failed = "failed";
|
||||
}
|
||||
namespace StellaOps.Concelier.Connector.Common;
|
||||
|
||||
/// <summary>
|
||||
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
|
||||
/// </summary>
|
||||
public static class DocumentStatuses
|
||||
{
|
||||
/// <summary>
|
||||
/// Document captured from the upstream source and awaiting schema validation/parsing.
|
||||
/// </summary>
|
||||
public const string PendingParse = "pending-parse";
|
||||
|
||||
/// <summary>
|
||||
/// Document parsed and sanitized; awaiting canonical mapping.
|
||||
/// </summary>
|
||||
public const string PendingMap = "pending-map";
|
||||
|
||||
/// <summary>
|
||||
/// Document fully mapped to canonical advisories.
|
||||
/// </summary>
|
||||
public const string Mapped = "mapped";
|
||||
|
||||
/// <summary>
|
||||
/// Document failed processing; requires manual intervention before retry.
|
||||
/// </summary>
|
||||
public const string Failed = "failed";
|
||||
}
|
||||
|
||||
@@ -1,43 +1,43 @@
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
|
||||
/// </summary>
|
||||
public sealed class CryptoJitterSource : IJitterSource
|
||||
{
|
||||
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
|
||||
{
|
||||
if (maxInclusive < minInclusive)
|
||||
{
|
||||
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
|
||||
}
|
||||
|
||||
if (minInclusive < TimeSpan.Zero)
|
||||
{
|
||||
minInclusive = TimeSpan.Zero;
|
||||
}
|
||||
|
||||
if (maxInclusive == minInclusive)
|
||||
{
|
||||
return minInclusive;
|
||||
}
|
||||
|
||||
var minTicks = minInclusive.Ticks;
|
||||
var maxTicks = maxInclusive.Ticks;
|
||||
var range = maxTicks - minTicks;
|
||||
|
||||
Span<byte> buffer = stackalloc byte[8];
|
||||
RandomNumberGenerator.Fill(buffer);
|
||||
var sample = BitConverter.ToUInt64(buffer);
|
||||
var ratio = sample / (double)ulong.MaxValue;
|
||||
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
|
||||
if (jitterTicks > range)
|
||||
{
|
||||
jitterTicks = range;
|
||||
}
|
||||
|
||||
return TimeSpan.FromTicks(minTicks + jitterTicks);
|
||||
}
|
||||
}
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
|
||||
/// </summary>
|
||||
public sealed class CryptoJitterSource : IJitterSource
|
||||
{
|
||||
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
|
||||
{
|
||||
if (maxInclusive < minInclusive)
|
||||
{
|
||||
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
|
||||
}
|
||||
|
||||
if (minInclusive < TimeSpan.Zero)
|
||||
{
|
||||
minInclusive = TimeSpan.Zero;
|
||||
}
|
||||
|
||||
if (maxInclusive == minInclusive)
|
||||
{
|
||||
return minInclusive;
|
||||
}
|
||||
|
||||
var minTicks = minInclusive.Ticks;
|
||||
var maxTicks = maxInclusive.Ticks;
|
||||
var range = maxTicks - minTicks;
|
||||
|
||||
Span<byte> buffer = stackalloc byte[8];
|
||||
RandomNumberGenerator.Fill(buffer);
|
||||
var sample = BitConverter.ToUInt64(buffer);
|
||||
var ratio = sample / (double)ulong.MaxValue;
|
||||
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
|
||||
if (jitterTicks > range)
|
||||
{
|
||||
jitterTicks = range;
|
||||
}
|
||||
|
||||
return TimeSpan.FromTicks(minTicks + jitterTicks);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Produces random jitter durations used to decorrelate retries.
|
||||
/// </summary>
|
||||
public interface IJitterSource
|
||||
{
|
||||
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
|
||||
}
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Produces random jitter durations used to decorrelate retries.
|
||||
/// </summary>
|
||||
public interface IJitterSource
|
||||
{
|
||||
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
|
||||
}
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Result of fetching raw response content without persisting a document.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchContentResult
|
||||
{
|
||||
private SourceFetchContentResult(
|
||||
HttpStatusCode statusCode,
|
||||
byte[]? content,
|
||||
bool notModified,
|
||||
string? etag,
|
||||
DateTimeOffset? lastModified,
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Result of fetching raw response content without persisting a document.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchContentResult
|
||||
{
|
||||
private SourceFetchContentResult(
|
||||
HttpStatusCode statusCode,
|
||||
byte[]? content,
|
||||
bool notModified,
|
||||
string? etag,
|
||||
DateTimeOffset? lastModified,
|
||||
string? contentType,
|
||||
int attempts,
|
||||
IReadOnlyDictionary<string, string>? headers)
|
||||
@@ -30,14 +30,14 @@ public sealed record SourceFetchContentResult
|
||||
public HttpStatusCode StatusCode { get; }
|
||||
|
||||
public byte[]? Content { get; }
|
||||
|
||||
public bool IsSuccess => Content is not null;
|
||||
|
||||
public bool IsNotModified { get; }
|
||||
|
||||
public string? ETag { get; }
|
||||
|
||||
public DateTimeOffset? LastModified { get; }
|
||||
|
||||
public bool IsSuccess => Content is not null;
|
||||
|
||||
public bool IsNotModified { get; }
|
||||
|
||||
public string? ETag { get; }
|
||||
|
||||
public DateTimeOffset? LastModified { get; }
|
||||
|
||||
public string? ContentType { get; }
|
||||
|
||||
|
||||
@@ -1,24 +1,24 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Parameters describing a fetch operation for a source connector.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchRequest(
|
||||
string ClientName,
|
||||
string SourceName,
|
||||
HttpMethod Method,
|
||||
Uri RequestUri,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null,
|
||||
string? ETag = null,
|
||||
DateTimeOffset? LastModified = null,
|
||||
TimeSpan? TimeoutOverride = null,
|
||||
IReadOnlyList<string>? AcceptHeaders = null)
|
||||
{
|
||||
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
|
||||
: this(clientName, sourceName, HttpMethod.Get, requestUri)
|
||||
{
|
||||
}
|
||||
}
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Parameters describing a fetch operation for a source connector.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchRequest(
|
||||
string ClientName,
|
||||
string SourceName,
|
||||
HttpMethod Method,
|
||||
Uri RequestUri,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null,
|
||||
string? ETag = null,
|
||||
DateTimeOffset? LastModified = null,
|
||||
TimeSpan? TimeoutOverride = null,
|
||||
IReadOnlyList<string>? AcceptHeaders = null)
|
||||
{
|
||||
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
|
||||
: this(clientName, sourceName, HttpMethod.Get, requestUri)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,34 +1,34 @@
|
||||
using System.Net;
|
||||
using StellaOps.Concelier.Storage.Contracts;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of fetching a raw document from an upstream source.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchResult
|
||||
{
|
||||
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
Document = document;
|
||||
IsNotModified = notModified;
|
||||
}
|
||||
|
||||
public HttpStatusCode StatusCode { get; }
|
||||
|
||||
public StorageDocument? Document { get; }
|
||||
|
||||
public bool IsSuccess => Document is not null;
|
||||
|
||||
public bool IsNotModified { get; }
|
||||
|
||||
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
|
||||
=> new(statusCode, document, notModified: false);
|
||||
|
||||
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
|
||||
=> new(statusCode, null, notModified: true);
|
||||
|
||||
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
|
||||
=> new(statusCode, null, notModified: false);
|
||||
}
|
||||
using System.Net;
|
||||
using StellaOps.Concelier.Storage.Contracts;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of fetching a raw document from an upstream source.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchResult
|
||||
{
|
||||
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
Document = document;
|
||||
IsNotModified = notModified;
|
||||
}
|
||||
|
||||
public HttpStatusCode StatusCode { get; }
|
||||
|
||||
public StorageDocument? Document { get; }
|
||||
|
||||
public bool IsSuccess => Document is not null;
|
||||
|
||||
public bool IsNotModified { get; }
|
||||
|
||||
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
|
||||
=> new(statusCode, document, notModified: false);
|
||||
|
||||
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
|
||||
=> new(statusCode, null, notModified: true);
|
||||
|
||||
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
|
||||
=> new(statusCode, null, notModified: false);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,10 +2,10 @@ using System.Globalization;
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Provides retry/backoff behavior for source HTTP fetches.
|
||||
/// </summary>
|
||||
|
||||
/// <summary>
|
||||
/// Provides retry/backoff behavior for source HTTP fetches.
|
||||
/// </summary>
|
||||
internal static class SourceRetryPolicy
|
||||
{
|
||||
private static readonly StringComparer HeaderComparer = StringComparer.OrdinalIgnoreCase;
|
||||
@@ -15,34 +15,34 @@ internal static class SourceRetryPolicy
|
||||
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> sender,
|
||||
int maxAttempts,
|
||||
TimeSpan baseDelay,
|
||||
IJitterSource jitterSource,
|
||||
Action<SourceRetryAttemptContext>? onRetry,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(requestFactory);
|
||||
ArgumentNullException.ThrowIfNull(sender);
|
||||
ArgumentNullException.ThrowIfNull(jitterSource);
|
||||
|
||||
var attempt = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
attempt++;
|
||||
using var request = requestFactory();
|
||||
HttpResponseMessage response;
|
||||
|
||||
try
|
||||
{
|
||||
response = await sender(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (attempt < maxAttempts)
|
||||
{
|
||||
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
|
||||
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
IJitterSource jitterSource,
|
||||
Action<SourceRetryAttemptContext>? onRetry,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(requestFactory);
|
||||
ArgumentNullException.ThrowIfNull(sender);
|
||||
ArgumentNullException.ThrowIfNull(jitterSource);
|
||||
|
||||
var attempt = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
attempt++;
|
||||
using var request = requestFactory();
|
||||
HttpResponseMessage response;
|
||||
|
||||
try
|
||||
{
|
||||
response = await sender(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (attempt < maxAttempts)
|
||||
{
|
||||
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
|
||||
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (NeedsRetry(response) && attempt < maxAttempts)
|
||||
{
|
||||
var delay = ComputeDelay(
|
||||
@@ -55,11 +55,11 @@ internal static class SourceRetryPolicy
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool NeedsRetry(HttpResponseMessage response)
|
||||
{
|
||||
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
@@ -76,13 +76,13 @@ internal static class SourceRetryPolicy
|
||||
return status >= 500 && status < 600;
|
||||
}
|
||||
|
||||
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
|
||||
{
|
||||
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
|
||||
{
|
||||
return retryAfter.Value;
|
||||
}
|
||||
|
||||
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
|
||||
{
|
||||
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
|
||||
{
|
||||
return retryAfter.Value;
|
||||
}
|
||||
|
||||
var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1));
|
||||
var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250))
|
||||
?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user