stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnector.cs
View File

@@ -10,8 +10,8 @@ using System.Xml.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Bson.IO;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Documents.IO;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Fetch;
@@ -292,7 +292,7 @@ public sealed class AcscConnector : IFeedConnector
var dto = AcscFeedParser.Parse(rawBytes, metadata.FeedSlug, parsedAt, _htmlSanitizer);
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = BsonDocument.Parse(json);
var payload = DocumentObject.Parse(json);
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
var dtoRecord = existingDto is null
@@ -678,7 +678,7 @@ public sealed class AcscConnector : IFeedConnector
private Task UpdateCursorAsync(AcscCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

38
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs
View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "acsc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "acsc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
}
}

88
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs
View File

@@ -1,44 +1,44 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Acsc.Configuration;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:acsc";
private const string FetchCron = "7,37 * * * *";
private const string ParseCron = "12,42 * * * *";
private const string MapCron = "17,47 * * * *";
private const string ProbeCron = "25,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAcscConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
return services;
}
}
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Acsc.Configuration;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:acsc";
private const string FetchCron = "7,37 * * * *";
private const string ParseCron = "12,42 * * * *";
private const string MapCron = "17,47 * * * *";
private const string ProbeCron = "25,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAcscConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
return services;
}
}

112
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscServiceCollectionExtensions.cs
View File

@@ -1,56 +1,56 @@
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.Acsc;
public static class AcscServiceCollectionExtensions
{
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AcscOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = options.UserAgent;
clientOptions.RequestVersion = options.RequestVersion;
clientOptions.VersionPolicy = options.VersionPolicy;
clientOptions.AllowAutoRedirect = true;
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.AllowAutoRedirect = true;
};
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
if (options.RelayEndpoint is not null)
{
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
});
services.AddSingleton<AcscDiagnostics>();
services.AddTransient<AcscConnector>();
return services;
}
}
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.Acsc;
public static class AcscServiceCollectionExtensions
{
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AcscOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = options.UserAgent;
clientOptions.RequestVersion = options.RequestVersion;
clientOptions.VersionPolicy = options.VersionPolicy;
clientOptions.AllowAutoRedirect = true;
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.AllowAutoRedirect = true;
};
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
if (options.RelayEndpoint is not null)
{
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
});
services.AddSingleton<AcscDiagnostics>();
services.AddTransient<AcscConnector>();
return services;
}
}

108
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Configuration/AcscFeedOptions.cs
View File

@@ -1,54 +1,54 @@
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Defines a single ACSC RSS feed endpoint.
/// </summary>
public sealed class AcscFeedOptions
{
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Logical slug for the feed (alerts, advisories, threats, etc.).
/// </summary>
public string Slug { get; set; } = "alerts";
/// <summary>
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
/// </summary>
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
/// <summary>
/// Indicates whether the feed is active.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Optional display name for logging.
/// </summary>
public string? DisplayName { get; set; }
internal void Validate(int index)
{
if (string.IsNullOrWhiteSpace(Slug))
{
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
}
if (!SlugPattern.IsMatch(Slug))
{
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
}
if (string.IsNullOrWhiteSpace(RelativePath))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
}
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
}
}
}
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Defines a single ACSC RSS feed endpoint.
/// </summary>
public sealed class AcscFeedOptions
{
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Logical slug for the feed (alerts, advisories, threats, etc.).
/// </summary>
public string Slug { get; set; } = "alerts";
/// <summary>
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
/// </summary>
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
/// <summary>
/// Indicates whether the feed is active.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Optional display name for logging.
/// </summary>
public string? DisplayName { get; set; }
internal void Validate(int index)
{
if (string.IsNullOrWhiteSpace(Slug))
{
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
}
if (!SlugPattern.IsMatch(Slug))
{
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
}
if (string.IsNullOrWhiteSpace(RelativePath))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
}
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
}
}
}

306
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Configuration/AcscOptions.cs
View File

@@ -1,153 +1,153 @@
using System.Net;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Connector options governing ACSC feed access and retry behaviour.
/// </summary>
public sealed class AcscOptions
{
public const string HttpClientName = "acsc";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
public AcscOptions()
{
Feeds = new List<AcscFeedOptions>
{
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
};
}
/// <summary>
/// Base endpoint for direct ACSC fetches.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
/// <summary>
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
/// </summary>
public Uri? RelayEndpoint { get; set; }
/// <summary>
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
/// </summary>
public bool PreferRelayByDefault { get; set; }
/// <summary>
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
/// </summary>
public bool EnableRelayFallback { get; set; } = true;
/// <summary>
/// If set, the connector will always use the relay endpoint and skip direct attempts.
/// </summary>
public bool ForceRelay { get; set; }
/// <summary>
/// Timeout applied to fetch requests (overrides HttpClient default).
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when marking fetch failures.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Look-back period used when deriving initial published cursors.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
/// <summary>
/// User-agent header sent with outbound requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
/// <summary>
/// RSS feeds requested during fetch.
/// </summary>
public IList<AcscFeedOptions> Feeds { get; }
/// <summary>
/// HTTP version policy requested for outbound requests.
/// </summary>
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
/// <summary>
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
/// </summary>
public Version RequestVersion { get; set; } = HttpVersion.Version20;
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
}
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
}
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
}
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
}
if (Feeds.Count == 0)
{
throw new InvalidOperationException("At least one ACSC feed must be configured.");
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < Feeds.Count; i++)
{
var feed = Feeds[i];
feed.Validate(i);
if (!feed.Enabled)
{
continue;
}
if (!seen.Add(feed.Slug))
{
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
}
}
}
}
using System.Net;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Connector options governing ACSC feed access and retry behaviour.
/// </summary>
public sealed class AcscOptions
{
public const string HttpClientName = "acsc";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
public AcscOptions()
{
Feeds = new List<AcscFeedOptions>
{
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
};
}
/// <summary>
/// Base endpoint for direct ACSC fetches.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
/// <summary>
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
/// </summary>
public Uri? RelayEndpoint { get; set; }
/// <summary>
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
/// </summary>
public bool PreferRelayByDefault { get; set; }
/// <summary>
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
/// </summary>
public bool EnableRelayFallback { get; set; } = true;
/// <summary>
/// If set, the connector will always use the relay endpoint and skip direct attempts.
/// </summary>
public bool ForceRelay { get; set; }
/// <summary>
/// Timeout applied to fetch requests (overrides HttpClient default).
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when marking fetch failures.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Look-back period used when deriving initial published cursors.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
/// <summary>
/// User-agent header sent with outbound requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
/// <summary>
/// RSS feeds requested during fetch.
/// </summary>
public IList<AcscFeedOptions> Feeds { get; }
/// <summary>
/// HTTP version policy requested for outbound requests.
/// </summary>
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
/// <summary>
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
/// </summary>
public Version RequestVersion { get; set; } = HttpVersion.Version20;
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
}
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
}
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
}
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
}
if (Feeds.Count == 0)
{
throw new InvalidOperationException("At least one ACSC feed must be configured.");
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < Feeds.Count; i++)
{
var feed = Feeds[i];
feed.Validate(i);
if (!feed.Enabled)
{
continue;
}
if (!seen.Add(feed.Slug))
{
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
}
}
}
}

282
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscCursor.cs
View File

@@ -1,141 +1,141 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal enum AcscEndpointPreference
{
Auto = 0,
Direct = 1,
Relay = 2,
}
internal sealed record AcscCursor(
AcscEndpointPreference PreferredEndpoint,
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
public static AcscCursor Empty { get; } = new(
AcscEndpointPreference.Auto,
EmptyFeedDictionary,
EmptyGuidList,
EmptyGuidList);
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
=> this with { PreferredEndpoint = preference };
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
{
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (values is not null)
{
foreach (var kvp in values)
{
snapshot[kvp.Key] = kvp.Value;
}
}
return this with { LastPublishedByFeed = snapshot };
}
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["preferredEndpoint"] = PreferredEndpoint.ToString(),
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
var feedsDocument = new BsonDocument();
foreach (var kvp in LastPublishedByFeed)
{
if (kvp.Value.HasValue)
{
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
}
}
document["feeds"] = feedsDocument;
return document;
}
public static AcscCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
? ParseEndpointPreference(endpointValue.AsString)
: AcscEndpointPreference.Auto;
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is BsonDocument feedsDocument)
{
foreach (var element in feedsDocument.Elements)
{
feeds[element.Name] = ParseDate(element.Value);
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AcscCursor(
preferredEndpoint,
feeds,
pendingDocuments,
pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
private static AcscEndpointPreference ParseEndpointPreference(string? value)
{
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
return AcscEndpointPreference.Auto;
}
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal enum AcscEndpointPreference
{
Auto = 0,
Direct = 1,
Relay = 2,
}
internal sealed record AcscCursor(
AcscEndpointPreference PreferredEndpoint,
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
public static AcscCursor Empty { get; } = new(
AcscEndpointPreference.Auto,
EmptyFeedDictionary,
EmptyGuidList,
EmptyGuidList);
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
=> this with { PreferredEndpoint = preference };
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
{
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (values is not null)
{
foreach (var kvp in values)
{
snapshot[kvp.Key] = kvp.Value;
}
}
return this with { LastPublishedByFeed = snapshot };
}
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["preferredEndpoint"] = PreferredEndpoint.ToString(),
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
var feedsDocument = new DocumentObject();
foreach (var kvp in LastPublishedByFeed)
{
if (kvp.Value.HasValue)
{
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
}
}
document["feeds"] = feedsDocument;
return document;
}
public static AcscCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
? ParseEndpointPreference(endpointValue.AsString)
: AcscEndpointPreference.Auto;
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is DocumentObject feedsDocument)
{
foreach (var element in feedsDocument.Elements)
{
feeds[element.Name] = ParseDate(element.Value);
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AcscCursor(
preferredEndpoint,
feeds,
pendingDocuments,
pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
private static AcscEndpointPreference ParseEndpointPreference(string? value)
{
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
return AcscEndpointPreference.Auto;
}
}

194
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscDiagnostics.cs
View File

@@ -1,97 +1,97 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
public sealed class AcscDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFallbacks;
private readonly Counter<long> _cursorUpdates;
private readonly Counter<long> _parseAttempts;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
public AcscDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
}
public void FetchAttempt(string feed, string mode)
=> _fetchAttempts.Add(1, GetTags(feed, mode));
public void FetchSuccess(string feed, string mode)
=> _fetchSuccess.Add(1, GetTags(feed, mode));
public void FetchFailure(string feed, string mode)
=> _fetchFailures.Add(1, GetTags(feed, mode));
public void FetchUnchanged(string feed, string mode)
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
public void FetchFallback(string feed, string mode, string reason)
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
public void CursorUpdated(string feed)
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseAttempt(string feed)
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseSuccess(string feed)
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseFailure(string feed, string reason)
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
{
new("feed", feed),
new("reason", reason),
});
public void MapSuccess(int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mapSuccess.Add(advisoryCount);
}
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
};
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
extra,
};
public void Dispose()
{
_meter.Dispose();
}
}
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
public sealed class AcscDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFallbacks;
private readonly Counter<long> _cursorUpdates;
private readonly Counter<long> _parseAttempts;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
public AcscDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
}
public void FetchAttempt(string feed, string mode)
=> _fetchAttempts.Add(1, GetTags(feed, mode));
public void FetchSuccess(string feed, string mode)
=> _fetchSuccess.Add(1, GetTags(feed, mode));
public void FetchFailure(string feed, string mode)
=> _fetchFailures.Add(1, GetTags(feed, mode));
public void FetchUnchanged(string feed, string mode)
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
public void FetchFallback(string feed, string mode, string reason)
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
public void CursorUpdated(string feed)
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseAttempt(string feed)
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseSuccess(string feed)
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseFailure(string feed, string reason)
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
{
new("feed", feed),
new("reason", reason),
});
public void MapSuccess(int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mapSuccess.Add(advisoryCount);
}
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
};
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
extra,
};
public void Dispose()
{
_meter.Dispose();
}
}

40
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscDocumentMetadata.cs
View File

@@ -1,20 +1,20 @@
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
{
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
{
if (document.Metadata is null)
{
return new AcscDocumentMetadata(string.Empty, string.Empty);
}
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
return new AcscDocumentMetadata(
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
}
}
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
{
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
{
if (document.Metadata is null)
{
return new AcscDocumentMetadata(string.Empty, string.Empty);
}
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
return new AcscDocumentMetadata(
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
}
}

116
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscDto.cs
View File

@@ -1,58 +1,58 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal sealed record AcscFeedDto(
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
[property: JsonPropertyName("feedLink")] string? FeedLink,
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
{
public static AcscFeedDto Empty { get; } = new(
FeedSlug: string.Empty,
FeedTitle: null,
FeedLink: null,
FeedUpdated: null,
ParsedAt: DateTimeOffset.UnixEpoch,
Entries: Array.Empty<AcscEntryDto>());
}
internal sealed record AcscEntryDto(
[property: JsonPropertyName("entryId")] string EntryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("link")] string? Link,
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("published")] DateTimeOffset? Published,
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
[property: JsonPropertyName("summary")] string Summary,
[property: JsonPropertyName("contentHtml")] string ContentHtml,
[property: JsonPropertyName("contentText")] string ContentText,
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
{
public static AcscEntryDto Empty { get; } = new(
EntryId: string.Empty,
Title: string.Empty,
Link: null,
FeedSlug: string.Empty,
Published: null,
Updated: null,
Summary: string.Empty,
ContentHtml: string.Empty,
ContentText: string.Empty,
References: Array.Empty<AcscReferenceDto>(),
Aliases: Array.Empty<string>(),
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
internal sealed record AcscReferenceDto(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url)
{
public static AcscReferenceDto Empty { get; } = new(
Title: string.Empty,
Url: string.Empty);
}
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal sealed record AcscFeedDto(
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
[property: JsonPropertyName("feedLink")] string? FeedLink,
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
{
public static AcscFeedDto Empty { get; } = new(
FeedSlug: string.Empty,
FeedTitle: null,
FeedLink: null,
FeedUpdated: null,
ParsedAt: DateTimeOffset.UnixEpoch,
Entries: Array.Empty<AcscEntryDto>());
}
internal sealed record AcscEntryDto(
[property: JsonPropertyName("entryId")] string EntryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("link")] string? Link,
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("published")] DateTimeOffset? Published,
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
[property: JsonPropertyName("summary")] string Summary,
[property: JsonPropertyName("contentHtml")] string ContentHtml,
[property: JsonPropertyName("contentText")] string ContentText,
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
{
public static AcscEntryDto Empty { get; } = new(
EntryId: string.Empty,
Title: string.Empty,
Link: null,
FeedSlug: string.Empty,
Published: null,
Updated: null,
Summary: string.Empty,
ContentHtml: string.Empty,
ContentText: string.Empty,
References: Array.Empty<AcscReferenceDto>(),
Aliases: Array.Empty<string>(),
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
internal sealed record AcscReferenceDto(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url)
{
public static AcscReferenceDto Empty { get; } = new(
Title: string.Empty,
Url: string.Empty);
}

1188
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscFeedParser.cs
View File

File diff suppressed because it is too large Load Diff

624
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscMapper.cs
View File

@@ -1,312 +1,312 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal static class AcscMapper
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<Advisory> Map(
AcscFeedDto feed,
DocumentRecord document,
DtoRecord dtoRecord,
string sourceName,
DateTimeOffset mappedAt)
{
ArgumentNullException.ThrowIfNull(feed);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
if (feed.Entries is null || feed.Entries.Count == 0)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(feed.Entries.Count);
foreach (var entry in feed.Entries)
{
if (entry is null)
{
continue;
}
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
var fetchProvenance = new AdvisoryProvenance(
sourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var feedProvenance = new AdvisoryProvenance(
sourceName,
"feed",
feed.FeedSlug ?? string.Empty,
feed.ParsedAt.ToUniversalTime(),
fieldMask: new[] { "summary" });
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
var provenance = new[]
{
fetchProvenance,
feedProvenance,
mappingProvenance,
};
var aliases = BuildAliases(entry);
var severity = TryGetSeverity(entry.Fields);
var references = BuildReferences(entry, sourceName, mappedAt);
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
var advisory = new Advisory(
advisoryKey,
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
language: "en",
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
modified: entry.Updated?.ToUniversalTime(),
severity: severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
advisories.Add(advisory);
}
return advisories;
}
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(entry.EntryId))
{
aliases.Add(entry.EntryId.Trim());
}
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
{
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias.Trim());
}
}
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
return aliases.Count == 0
? Array.Empty<string>()
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Validation.LooksLikeHttpUrl(url))
{
return;
}
if (!seen.Add(url))
{
return;
}
references.Add(new AdvisoryReference(
url,
kind,
sourceTag,
summary,
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
}
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
{
if (reference is null)
{
continue;
}
AddReference(reference.Url, "reference", null, reference.Title);
}
return references.Count == 0
? Array.Empty<AdvisoryReference>()
: references
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
if (entry.Fields is null || entry.Fields.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
if (string.IsNullOrWhiteSpace(systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
var identifiers = systemsAffected
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(static value => value.Trim())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (identifiers.Length == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(identifiers.Length);
foreach (var identifier in identifiers)
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
{
if (fields is null || fields.Count == 0)
{
return null;
}
var keys = new[]
{
"severity",
"riskLevel",
"threatLevel",
"impact",
};
foreach (var key in keys)
{
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
{
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
? entry.EntryId
: !string.IsNullOrWhiteSpace(entry.Link)
? entry.Link
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
}
return $"{sourceName}/{slug}/{identifier}";
}
private static string ToSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var builder = new StringBuilder(value.Length);
var previousDash = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
previousDash = false;
}
else if (!previousDash)
{
builder.Append('-');
previousDash = true;
}
}
var slug = builder.ToString().Trim('-');
if (string.IsNullOrEmpty(slug))
{
slug = CreateHash(value);
}
return slug.Length <= 64 ? slug : slug[..64];
}
private static string CreateHash(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
}
}
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal static class AcscMapper
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<Advisory> Map(
AcscFeedDto feed,
DocumentRecord document,
DtoRecord dtoRecord,
string sourceName,
DateTimeOffset mappedAt)
{
ArgumentNullException.ThrowIfNull(feed);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
if (feed.Entries is null || feed.Entries.Count == 0)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(feed.Entries.Count);
foreach (var entry in feed.Entries)
{
if (entry is null)
{
continue;
}
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
var fetchProvenance = new AdvisoryProvenance(
sourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var feedProvenance = new AdvisoryProvenance(
sourceName,
"feed",
feed.FeedSlug ?? string.Empty,
feed.ParsedAt.ToUniversalTime(),
fieldMask: new[] { "summary" });
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
var provenance = new[]
{
fetchProvenance,
feedProvenance,
mappingProvenance,
};
var aliases = BuildAliases(entry);
var severity = TryGetSeverity(entry.Fields);
var references = BuildReferences(entry, sourceName, mappedAt);
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
var advisory = new Advisory(
advisoryKey,
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
language: "en",
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
modified: entry.Updated?.ToUniversalTime(),
severity: severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
advisories.Add(advisory);
}
return advisories;
}
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(entry.EntryId))
{
aliases.Add(entry.EntryId.Trim());
}
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
{
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias.Trim());
}
}
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
return aliases.Count == 0
? Array.Empty<string>()
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Validation.LooksLikeHttpUrl(url))
{
return;
}
if (!seen.Add(url))
{
return;
}
references.Add(new AdvisoryReference(
url,
kind,
sourceTag,
summary,
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
}
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
{
if (reference is null)
{
continue;
}
AddReference(reference.Url, "reference", null, reference.Title);
}
return references.Count == 0
? Array.Empty<AdvisoryReference>()
: references
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
if (entry.Fields is null || entry.Fields.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
if (string.IsNullOrWhiteSpace(systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
var identifiers = systemsAffected
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(static value => value.Trim())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (identifiers.Length == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(identifiers.Length);
foreach (var identifier in identifiers)
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
{
if (fields is null || fields.Count == 0)
{
return null;
}
var keys = new[]
{
"severity",
"riskLevel",
"threatLevel",
"impact",
};
foreach (var key in keys)
{
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
{
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
? entry.EntryId
: !string.IsNullOrWhiteSpace(entry.Link)
? entry.Link
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
}
return $"{sourceName}/{slug}/{identifier}";
}
private static string ToSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var builder = new StringBuilder(value.Length);
var previousDash = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
previousDash = false;
}
else if (!previousDash)
{
builder.Append('-');
previousDash = true;
}
}
var slug = builder.ToString().Trim('-');
if (string.IsNullOrEmpty(slug))
{
slug = CreateHash(value);
}
return slug.Length <= 64 ? slug : slug[..64];
}
private static string CreateHash(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
}
}

110
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Jobs.cs
View File

@@ -1,55 +1,55 @@
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Acsc;
internal static class AcscJobKinds
{
public const string Fetch = "source:acsc:fetch";
public const string Parse = "source:acsc:parse";
public const string Map = "source:acsc:map";
public const string Probe = "source:acsc:probe";
}
internal sealed class AcscFetchJob : IJob
{
private readonly AcscConnector _connector;
public AcscFetchJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AcscParseJob : IJob
{
private readonly AcscConnector _connector;
public AcscParseJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AcscMapJob : IJob
{
private readonly AcscConnector _connector;
public AcscMapJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
internal sealed class AcscProbeJob : IJob
{
private readonly AcscConnector _connector;
public AcscProbeJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ProbeAsync(cancellationToken);
}
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Acsc;
internal static class AcscJobKinds
{
public const string Fetch = "source:acsc:fetch";
public const string Parse = "source:acsc:parse";
public const string Map = "source:acsc:map";
public const string Probe = "source:acsc:probe";
}
internal sealed class AcscFetchJob : IJob
{
private readonly AcscConnector _connector;
public AcscFetchJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AcscParseJob : IJob
{
private readonly AcscConnector _connector;
public AcscParseJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AcscMapJob : IJob
{
private readonly AcscConnector _connector;
public AcscMapJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
internal sealed class AcscProbeJob : IJob
{
private readonly AcscConnector _connector;
public AcscProbeJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ProbeAsync(cancellationToken);
}

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Properties/AssemblyInfo.cs
View File

@@ -1,4 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]