stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnector.cs
View File

@@ -10,8 +10,8 @@ using System.Xml.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Bson.IO;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Documents.IO;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Fetch;
@@ -292,7 +292,7 @@ public sealed class AcscConnector : IFeedConnector
var dto = AcscFeedParser.Parse(rawBytes, metadata.FeedSlug, parsedAt, _htmlSanitizer);
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = BsonDocument.Parse(json);
var payload = DocumentObject.Parse(json);
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
var dtoRecord = existingDto is null
@@ -678,7 +678,7 @@ public sealed class AcscConnector : IFeedConnector
private Task UpdateCursorAsync(AcscCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

38
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs
View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "acsc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "acsc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<AcscConnector>(services);
}
}

88
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs
View File

@@ -1,44 +1,44 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Acsc.Configuration;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:acsc";
private const string FetchCron = "7,37 * * * *";
private const string ParseCron = "12,42 * * * *";
private const string MapCron = "17,47 * * * *";
private const string ProbeCron = "25,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAcscConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
return services;
}
}
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Acsc.Configuration;
namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:acsc";
private const string FetchCron = "7,37 * * * *";
private const string ParseCron = "12,42 * * * *";
private const string MapCron = "17,47 * * * *";
private const string ProbeCron = "25,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAcscConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<AcscFetchJob>(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration)
.AddJob<AcscParseJob>(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration)
.AddJob<AcscMapJob>(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration)
.AddJob<AcscProbeJob>(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration);
return services;
}
}

112
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscServiceCollectionExtensions.cs
View File

@@ -1,56 +1,56 @@
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.Acsc;
public static class AcscServiceCollectionExtensions
{
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AcscOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = options.UserAgent;
clientOptions.RequestVersion = options.RequestVersion;
clientOptions.VersionPolicy = options.VersionPolicy;
clientOptions.AllowAutoRedirect = true;
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.AllowAutoRedirect = true;
};
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
if (options.RelayEndpoint is not null)
{
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
});
services.AddSingleton<AcscDiagnostics>();
services.AddTransient<AcscConnector>();
return services;
}
}
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.Acsc;
public static class AcscServiceCollectionExtensions
{
public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action<AcscOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AcscOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AcscOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = options.UserAgent;
clientOptions.RequestVersion = options.RequestVersion;
clientOptions.VersionPolicy = options.VersionPolicy;
clientOptions.AllowAutoRedirect = true;
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
handler.AllowAutoRedirect = true;
};
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
if (options.RelayEndpoint is not null)
{
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
});
services.AddSingleton<AcscDiagnostics>();
services.AddTransient<AcscConnector>();
return services;
}
}

108
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Configuration/AcscFeedOptions.cs
View File

@@ -1,54 +1,54 @@
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Defines a single ACSC RSS feed endpoint.
/// </summary>
public sealed class AcscFeedOptions
{
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Logical slug for the feed (alerts, advisories, threats, etc.).
/// </summary>
public string Slug { get; set; } = "alerts";
/// <summary>
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
/// </summary>
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
/// <summary>
/// Indicates whether the feed is active.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Optional display name for logging.
/// </summary>
public string? DisplayName { get; set; }
internal void Validate(int index)
{
if (string.IsNullOrWhiteSpace(Slug))
{
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
}
if (!SlugPattern.IsMatch(Slug))
{
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
}
if (string.IsNullOrWhiteSpace(RelativePath))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
}
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
}
}
}
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Defines a single ACSC RSS feed endpoint.
/// </summary>
public sealed class AcscFeedOptions
{
private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Logical slug for the feed (alerts, advisories, threats, etc.).
/// </summary>
public string Slug { get; set; } = "alerts";
/// <summary>
/// Relative path (under <see cref="AcscOptions.BaseEndpoint"/>) for the RSS feed.
/// </summary>
public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss";
/// <summary>
/// Indicates whether the feed is active.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Optional display name for logging.
/// </summary>
public string? DisplayName { get; set; }
internal void Validate(int index)
{
if (string.IsNullOrWhiteSpace(Slug))
{
throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug.");
}
if (!SlugPattern.IsMatch(Slug))
{
throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators.");
}
if (string.IsNullOrWhiteSpace(RelativePath))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path.");
}
if (!RelativePath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}').");
}
}
}

306
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Configuration/AcscOptions.cs
View File

@@ -1,153 +1,153 @@
using System.Net;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Connector options governing ACSC feed access and retry behaviour.
/// </summary>
public sealed class AcscOptions
{
public const string HttpClientName = "acsc";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
public AcscOptions()
{
Feeds = new List<AcscFeedOptions>
{
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
};
}
/// <summary>
/// Base endpoint for direct ACSC fetches.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
/// <summary>
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
/// </summary>
public Uri? RelayEndpoint { get; set; }
/// <summary>
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
/// </summary>
public bool PreferRelayByDefault { get; set; }
/// <summary>
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
/// </summary>
public bool EnableRelayFallback { get; set; } = true;
/// <summary>
/// If set, the connector will always use the relay endpoint and skip direct attempts.
/// </summary>
public bool ForceRelay { get; set; }
/// <summary>
/// Timeout applied to fetch requests (overrides HttpClient default).
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when marking fetch failures.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Look-back period used when deriving initial published cursors.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
/// <summary>
/// User-agent header sent with outbound requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
/// <summary>
/// RSS feeds requested during fetch.
/// </summary>
public IList<AcscFeedOptions> Feeds { get; }
/// <summary>
/// HTTP version policy requested for outbound requests.
/// </summary>
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
/// <summary>
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
/// </summary>
public Version RequestVersion { get; set; } = HttpVersion.Version20;
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
}
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
}
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
}
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
}
if (Feeds.Count == 0)
{
throw new InvalidOperationException("At least one ACSC feed must be configured.");
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < Feeds.Count; i++)
{
var feed = Feeds[i];
feed.Validate(i);
if (!feed.Enabled)
{
continue;
}
if (!seen.Add(feed.Slug))
{
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
}
}
}
}
using System.Net;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Acsc.Configuration;
/// <summary>
/// Connector options governing ACSC feed access and retry behaviour.
/// </summary>
public sealed class AcscOptions
{
public const string HttpClientName = "acsc";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5);
private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120);
public AcscOptions()
{
Feeds = new List<AcscFeedOptions>
{
new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" },
new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" },
new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false },
new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false },
new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false },
};
}
/// <summary>
/// Base endpoint for direct ACSC fetches.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute);
/// <summary>
/// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections.
/// </summary>
public Uri? RelayEndpoint { get; set; }
/// <summary>
/// Default mode when no preference has been captured in connector state. When <c>true</c>, the relay will be preferred for initial fetches.
/// </summary>
public bool PreferRelayByDefault { get; set; }
/// <summary>
/// If enabled, the connector may switch to the relay endpoint when direct fetches fail.
/// </summary>
public bool EnableRelayFallback { get; set; } = true;
/// <summary>
/// If set, the connector will always use the relay endpoint and skip direct attempts.
/// </summary>
public bool ForceRelay { get; set; }
/// <summary>
/// Timeout applied to fetch requests (overrides HttpClient default).
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when marking fetch failures.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Look-back period used when deriving initial published cursors.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill;
/// <summary>
/// User-agent header sent with outbound requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
/// <summary>
/// RSS feeds requested during fetch.
/// </summary>
public IList<AcscFeedOptions> Feeds { get; }
/// <summary>
/// HTTP version policy requested for outbound requests.
/// </summary>
public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower;
/// <summary>
/// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade).
/// </summary>
public Version RequestVersion { get; set; } = HttpVersion.Version20;
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI.");
}
if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash.");
}
if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified.");
}
if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC FailureBackoff cannot be negative.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC InitialBackfill must be positive.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("ACSC UserAgent cannot be empty.");
}
if (Feeds.Count == 0)
{
throw new InvalidOperationException("At least one ACSC feed must be configured.");
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < Feeds.Count; i++)
{
var feed = Feeds[i];
feed.Validate(i);
if (!feed.Enabled)
{
continue;
}
if (!seen.Add(feed.Slug))
{
throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive).");
}
}
}
}

282
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscCursor.cs
View File

@@ -1,141 +1,141 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal enum AcscEndpointPreference
{
Auto = 0,
Direct = 1,
Relay = 2,
}
internal sealed record AcscCursor(
AcscEndpointPreference PreferredEndpoint,
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
public static AcscCursor Empty { get; } = new(
AcscEndpointPreference.Auto,
EmptyFeedDictionary,
EmptyGuidList,
EmptyGuidList);
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
=> this with { PreferredEndpoint = preference };
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
{
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (values is not null)
{
foreach (var kvp in values)
{
snapshot[kvp.Key] = kvp.Value;
}
}
return this with { LastPublishedByFeed = snapshot };
}
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["preferredEndpoint"] = PreferredEndpoint.ToString(),
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
var feedsDocument = new BsonDocument();
foreach (var kvp in LastPublishedByFeed)
{
if (kvp.Value.HasValue)
{
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
}
}
document["feeds"] = feedsDocument;
return document;
}
public static AcscCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
? ParseEndpointPreference(endpointValue.AsString)
: AcscEndpointPreference.Auto;
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is BsonDocument feedsDocument)
{
foreach (var element in feedsDocument.Elements)
{
feeds[element.Name] = ParseDate(element.Value);
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AcscCursor(
preferredEndpoint,
feeds,
pendingDocuments,
pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
private static AcscEndpointPreference ParseEndpointPreference(string? value)
{
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
return AcscEndpointPreference.Auto;
}
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal enum AcscEndpointPreference
{
Auto = 0,
Direct = 1,
Relay = 2,
}
internal sealed record AcscCursor(
AcscEndpointPreference PreferredEndpoint,
IReadOnlyDictionary<string, DateTimeOffset?> LastPublishedByFeed,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyFeedDictionary =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
public static AcscCursor Empty { get; } = new(
AcscEndpointPreference.Auto,
EmptyFeedDictionary,
EmptyGuidList,
EmptyGuidList);
public AcscCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList };
public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference)
=> this with { PreferredEndpoint = preference };
public AcscCursor WithLastPublished(IDictionary<string, DateTimeOffset?> values)
{
var snapshot = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (values is not null)
{
foreach (var kvp in values)
{
snapshot[kvp.Key] = kvp.Value;
}
}
return this with { LastPublishedByFeed = snapshot };
}
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["preferredEndpoint"] = PreferredEndpoint.ToString(),
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
var feedsDocument = new DocumentObject();
foreach (var kvp in LastPublishedByFeed)
{
if (kvp.Value.HasValue)
{
feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime;
}
}
document["feeds"] = feedsDocument;
return document;
}
public static AcscCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue)
? ParseEndpointPreference(endpointValue.AsString)
: AcscEndpointPreference.Auto;
var feeds = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is DocumentObject feedsDocument)
{
foreach (var element in feedsDocument.Elements)
{
feeds[element.Name] = ParseDate(element.Value);
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AcscCursor(
preferredEndpoint,
feeds,
pendingDocuments,
pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
private static AcscEndpointPreference ParseEndpointPreference(string? value)
{
if (Enum.TryParse<AcscEndpointPreference>(value, ignoreCase: true, out var parsed))
{
return parsed;
}
return AcscEndpointPreference.Auto;
}
}

194
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscDiagnostics.cs
View File

@@ -1,97 +1,97 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
public sealed class AcscDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFallbacks;
private readonly Counter<long> _cursorUpdates;
private readonly Counter<long> _parseAttempts;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
public AcscDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
}
public void FetchAttempt(string feed, string mode)
=> _fetchAttempts.Add(1, GetTags(feed, mode));
public void FetchSuccess(string feed, string mode)
=> _fetchSuccess.Add(1, GetTags(feed, mode));
public void FetchFailure(string feed, string mode)
=> _fetchFailures.Add(1, GetTags(feed, mode));
public void FetchUnchanged(string feed, string mode)
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
public void FetchFallback(string feed, string mode, string reason)
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
public void CursorUpdated(string feed)
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseAttempt(string feed)
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseSuccess(string feed)
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseFailure(string feed, string reason)
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
{
new("feed", feed),
new("reason", reason),
});
public void MapSuccess(int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mapSuccess.Add(advisoryCount);
}
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
};
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
extra,
};
public void Dispose()
{
_meter.Dispose();
}
}
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
public sealed class AcscDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Acsc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFallbacks;
private readonly Counter<long> _cursorUpdates;
private readonly Counter<long> _parseAttempts;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
public AcscDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("acsc.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("acsc.fetch.success", unit: "operations");
_fetchFailures = _meter.CreateCounter<long>("acsc.fetch.failures", unit: "operations");
_fetchUnchanged = _meter.CreateCounter<long>("acsc.fetch.unchanged", unit: "operations");
_fetchFallbacks = _meter.CreateCounter<long>("acsc.fetch.fallbacks", unit: "operations");
_cursorUpdates = _meter.CreateCounter<long>("acsc.cursor.published_updates", unit: "feeds");
_parseAttempts = _meter.CreateCounter<long>("acsc.parse.attempts", unit: "documents");
_parseSuccess = _meter.CreateCounter<long>("acsc.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("acsc.parse.failures", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("acsc.map.success", unit: "advisories");
}
public void FetchAttempt(string feed, string mode)
=> _fetchAttempts.Add(1, GetTags(feed, mode));
public void FetchSuccess(string feed, string mode)
=> _fetchSuccess.Add(1, GetTags(feed, mode));
public void FetchFailure(string feed, string mode)
=> _fetchFailures.Add(1, GetTags(feed, mode));
public void FetchUnchanged(string feed, string mode)
=> _fetchUnchanged.Add(1, GetTags(feed, mode));
public void FetchFallback(string feed, string mode, string reason)
=> _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair<string, object?>("reason", reason)));
public void CursorUpdated(string feed)
=> _cursorUpdates.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseAttempt(string feed)
=> _parseAttempts.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseSuccess(string feed)
=> _parseSuccess.Add(1, new KeyValuePair<string, object?>("feed", feed));
public void ParseFailure(string feed, string reason)
=> _parseFailures.Add(1, new KeyValuePair<string, object?>[]
{
new("feed", feed),
new("reason", reason),
});
public void MapSuccess(int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mapSuccess.Add(advisoryCount);
}
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
};
private static KeyValuePair<string, object?>[] GetTags(string feed, string mode, KeyValuePair<string, object?> extra)
=> new[]
{
new KeyValuePair<string, object?>("feed", feed),
new KeyValuePair<string, object?>("mode", mode),
extra,
};
public void Dispose()
{
_meter.Dispose();
}
}

40
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscDocumentMetadata.cs
View File

@@ -1,20 +1,20 @@
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
{
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
{
if (document.Metadata is null)
{
return new AcscDocumentMetadata(string.Empty, string.Empty);
}
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
return new AcscDocumentMetadata(
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
}
}
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode)
{
public static AcscDocumentMetadata FromDocument(DocumentRecord document)
{
if (document.Metadata is null)
{
return new AcscDocumentMetadata(string.Empty, string.Empty);
}
document.Metadata.TryGetValue("acsc.feed.slug", out var slug);
document.Metadata.TryGetValue("acsc.fetch.mode", out var mode);
return new AcscDocumentMetadata(
string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(),
string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim());
}
}

116
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscDto.cs
View File

@@ -1,58 +1,58 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal sealed record AcscFeedDto(
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
[property: JsonPropertyName("feedLink")] string? FeedLink,
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
{
public static AcscFeedDto Empty { get; } = new(
FeedSlug: string.Empty,
FeedTitle: null,
FeedLink: null,
FeedUpdated: null,
ParsedAt: DateTimeOffset.UnixEpoch,
Entries: Array.Empty<AcscEntryDto>());
}
internal sealed record AcscEntryDto(
[property: JsonPropertyName("entryId")] string EntryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("link")] string? Link,
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("published")] DateTimeOffset? Published,
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
[property: JsonPropertyName("summary")] string Summary,
[property: JsonPropertyName("contentHtml")] string ContentHtml,
[property: JsonPropertyName("contentText")] string ContentText,
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
{
public static AcscEntryDto Empty { get; } = new(
EntryId: string.Empty,
Title: string.Empty,
Link: null,
FeedSlug: string.Empty,
Published: null,
Updated: null,
Summary: string.Empty,
ContentHtml: string.Empty,
ContentText: string.Empty,
References: Array.Empty<AcscReferenceDto>(),
Aliases: Array.Empty<string>(),
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
internal sealed record AcscReferenceDto(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url)
{
public static AcscReferenceDto Empty { get; } = new(
Title: string.Empty,
Url: string.Empty);
}
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal sealed record AcscFeedDto(
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("feedTitle")] string? FeedTitle,
[property: JsonPropertyName("feedLink")] string? FeedLink,
[property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated,
[property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt,
[property: JsonPropertyName("entries")] IReadOnlyList<AcscEntryDto> Entries)
{
public static AcscFeedDto Empty { get; } = new(
FeedSlug: string.Empty,
FeedTitle: null,
FeedLink: null,
FeedUpdated: null,
ParsedAt: DateTimeOffset.UnixEpoch,
Entries: Array.Empty<AcscEntryDto>());
}
internal sealed record AcscEntryDto(
[property: JsonPropertyName("entryId")] string EntryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("link")] string? Link,
[property: JsonPropertyName("feedSlug")] string FeedSlug,
[property: JsonPropertyName("published")] DateTimeOffset? Published,
[property: JsonPropertyName("updated")] DateTimeOffset? Updated,
[property: JsonPropertyName("summary")] string Summary,
[property: JsonPropertyName("contentHtml")] string ContentHtml,
[property: JsonPropertyName("contentText")] string ContentText,
[property: JsonPropertyName("references")] IReadOnlyList<AcscReferenceDto> References,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
[property: JsonPropertyName("fields")] IReadOnlyDictionary<string, string> Fields)
{
public static AcscEntryDto Empty { get; } = new(
EntryId: string.Empty,
Title: string.Empty,
Link: null,
FeedSlug: string.Empty,
Published: null,
Updated: null,
Summary: string.Empty,
ContentHtml: string.Empty,
ContentText: string.Empty,
References: Array.Empty<AcscReferenceDto>(),
Aliases: Array.Empty<string>(),
Fields: new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
internal sealed record AcscReferenceDto(
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("url")] string Url)
{
public static AcscReferenceDto Empty { get; } = new(
Title: string.Empty,
Url: string.Empty);
}

1188
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscFeedParser.cs
View File

File diff suppressed because it is too large Load Diff

624
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscMapper.cs
View File

@@ -1,312 +1,312 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal static class AcscMapper
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<Advisory> Map(
AcscFeedDto feed,
DocumentRecord document,
DtoRecord dtoRecord,
string sourceName,
DateTimeOffset mappedAt)
{
ArgumentNullException.ThrowIfNull(feed);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
if (feed.Entries is null || feed.Entries.Count == 0)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(feed.Entries.Count);
foreach (var entry in feed.Entries)
{
if (entry is null)
{
continue;
}
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
var fetchProvenance = new AdvisoryProvenance(
sourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var feedProvenance = new AdvisoryProvenance(
sourceName,
"feed",
feed.FeedSlug ?? string.Empty,
feed.ParsedAt.ToUniversalTime(),
fieldMask: new[] { "summary" });
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
var provenance = new[]
{
fetchProvenance,
feedProvenance,
mappingProvenance,
};
var aliases = BuildAliases(entry);
var severity = TryGetSeverity(entry.Fields);
var references = BuildReferences(entry, sourceName, mappedAt);
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
var advisory = new Advisory(
advisoryKey,
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
language: "en",
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
modified: entry.Updated?.ToUniversalTime(),
severity: severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
advisories.Add(advisory);
}
return advisories;
}
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(entry.EntryId))
{
aliases.Add(entry.EntryId.Trim());
}
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
{
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias.Trim());
}
}
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
return aliases.Count == 0
? Array.Empty<string>()
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Validation.LooksLikeHttpUrl(url))
{
return;
}
if (!seen.Add(url))
{
return;
}
references.Add(new AdvisoryReference(
url,
kind,
sourceTag,
summary,
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
}
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
{
if (reference is null)
{
continue;
}
AddReference(reference.Url, "reference", null, reference.Title);
}
return references.Count == 0
? Array.Empty<AdvisoryReference>()
: references
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
if (entry.Fields is null || entry.Fields.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
if (string.IsNullOrWhiteSpace(systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
var identifiers = systemsAffected
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(static value => value.Trim())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (identifiers.Length == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(identifiers.Length);
foreach (var identifier in identifiers)
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
{
if (fields is null || fields.Count == 0)
{
return null;
}
var keys = new[]
{
"severity",
"riskLevel",
"threatLevel",
"impact",
};
foreach (var key in keys)
{
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
{
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
? entry.EntryId
: !string.IsNullOrWhiteSpace(entry.Link)
? entry.Link
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
}
return $"{sourceName}/{slug}/{identifier}";
}
private static string ToSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var builder = new StringBuilder(value.Length);
var previousDash = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
previousDash = false;
}
else if (!previousDash)
{
builder.Append('-');
previousDash = true;
}
}
var slug = builder.ToString().Trim('-');
if (string.IsNullOrEmpty(slug))
{
slug = CreateHash(value);
}
return slug.Length <= 64 ? slug : slug[..64];
}
private static string CreateHash(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
}
}
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
internal static class AcscMapper
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<Advisory> Map(
AcscFeedDto feed,
DocumentRecord document,
DtoRecord dtoRecord,
string sourceName,
DateTimeOffset mappedAt)
{
ArgumentNullException.ThrowIfNull(feed);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
if (feed.Entries is null || feed.Entries.Count == 0)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(feed.Entries.Count);
foreach (var entry in feed.Entries)
{
if (entry is null)
{
continue;
}
var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry);
var fetchProvenance = new AdvisoryProvenance(
sourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var feedProvenance = new AdvisoryProvenance(
sourceName,
"feed",
feed.FeedSlug ?? string.Empty,
feed.ParsedAt.ToUniversalTime(),
fieldMask: new[] { "summary" });
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
var provenance = new[]
{
fetchProvenance,
feedProvenance,
mappingProvenance,
};
var aliases = BuildAliases(entry);
var severity = TryGetSeverity(entry.Fields);
var references = BuildReferences(entry, sourceName, mappedAt);
var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt);
var advisory = new Advisory(
advisoryKey,
string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title,
string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary,
language: "en",
published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(),
modified: entry.Updated?.ToUniversalTime(),
severity: severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
advisories.Add(advisory);
}
return advisories;
}
private static IReadOnlyList<string> BuildAliases(AcscEntryDto entry)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(entry.EntryId))
{
aliases.Add(entry.EntryId.Trim());
}
foreach (var alias in entry.Aliases ?? Array.Empty<string>())
{
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias.Trim());
}
}
foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast<Match>())
{
var value = match.Value.ToUpperInvariant();
aliases.Add(value);
}
return aliases.Count == 0
? Array.Empty<string>()
: aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void AddReference(string? url, string? kind, string? sourceTag, string? summary)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Validation.LooksLikeHttpUrl(url))
{
return;
}
if (!seen.Add(url))
{
return;
}
references.Add(new AdvisoryReference(
url,
kind,
sourceTag,
summary,
new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())));
}
AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title);
foreach (var reference in entry.References ?? Array.Empty<AcscReferenceDto>())
{
if (reference is null)
{
continue;
}
AddReference(reference.Url, "reference", null, reference.Title);
}
return references.Count == 0
? Array.Empty<AdvisoryReference>()
: references
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt)
{
if (entry.Fields is null || entry.Fields.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
if (string.IsNullOrWhiteSpace(systemsAffected))
{
return Array.Empty<AffectedPackage>();
}
var identifiers = systemsAffected
.Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(static value => value.Trim())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (identifiers.Length == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(identifiers.Length);
foreach (var identifier in identifiers)
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? TryGetSeverity(IReadOnlyDictionary<string, string> fields)
{
if (fields is null || fields.Count == 0)
{
return null;
}
var keys = new[]
{
"severity",
"riskLevel",
"threatLevel",
"impact",
};
foreach (var key in keys)
{
if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry)
{
var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug);
var candidate = !string.IsNullOrWhiteSpace(entry.EntryId)
? entry.EntryId
: !string.IsNullOrWhiteSpace(entry.Link)
? entry.Link
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
}
return $"{sourceName}/{slug}/{identifier}";
}
private static string ToSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var builder = new StringBuilder(value.Length);
var previousDash = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
previousDash = false;
}
else if (!previousDash)
{
builder.Append('-');
previousDash = true;
}
}
var slug = builder.ToString().Trim('-');
if (string.IsNullOrEmpty(slug))
{
slug = CreateHash(value);
}
return slug.Length <= 64 ? slug : slug[..64];
}
private static string CreateHash(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant()[..16];
}
}

110
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Jobs.cs
View File

@@ -1,55 +1,55 @@
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Acsc;
internal static class AcscJobKinds
{
public const string Fetch = "source:acsc:fetch";
public const string Parse = "source:acsc:parse";
public const string Map = "source:acsc:map";
public const string Probe = "source:acsc:probe";
}
internal sealed class AcscFetchJob : IJob
{
private readonly AcscConnector _connector;
public AcscFetchJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AcscParseJob : IJob
{
private readonly AcscConnector _connector;
public AcscParseJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AcscMapJob : IJob
{
private readonly AcscConnector _connector;
public AcscMapJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
internal sealed class AcscProbeJob : IJob
{
private readonly AcscConnector _connector;
public AcscProbeJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ProbeAsync(cancellationToken);
}
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Acsc;
internal static class AcscJobKinds
{
public const string Fetch = "source:acsc:fetch";
public const string Parse = "source:acsc:parse";
public const string Map = "source:acsc:map";
public const string Probe = "source:acsc:probe";
}
internal sealed class AcscFetchJob : IJob
{
private readonly AcscConnector _connector;
public AcscFetchJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AcscParseJob : IJob
{
private readonly AcscConnector _connector;
public AcscParseJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AcscMapJob : IJob
{
private readonly AcscConnector _connector;
public AcscMapJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
internal sealed class AcscProbeJob : IJob
{
private readonly AcscConnector _connector;
public AcscProbeJob(AcscConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ProbeAsync(cancellationToken);
}

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Properties/AssemblyInfo.cs
View File

@@ -1,4 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")]

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsConnector.cs
View File

@@ -10,7 +10,7 @@ using System.Threading.Tasks;
using System.Globalization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Cccs.Internal;
using StellaOps.Concelier.Connector.Common;
@@ -332,7 +332,7 @@ public sealed class CccsConnector : IFeedConnector
}
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var dtoBson = BsonDocument.Parse(dtoJson);
var dtoBson = DocumentObject.Parse(dtoJson);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -464,7 +464,7 @@ public sealed class CccsConnector : IFeedConnector
private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

42
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsConnectorPlugin.cs
View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cccs";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CccsConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CccsConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cccs";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CccsConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CccsConnector>();
}
}

100
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs
View File

@@ -1,50 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Cccs.Configuration;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cccs";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCccsConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CccsFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Cccs.Configuration;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cccs";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCccsConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CccsFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

94
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsServiceCollectionExtensions.cs
View File

@@ -1,47 +1,47 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Cccs.Internal;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs;
public static class CccsServiceCollectionExtensions
{
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CccsOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0";
clientOptions.Timeout = options.RequestTimeout;
clientOptions.AllowedHosts.Clear();
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
{
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
}
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
clientOptions.AllowedHosts.Add("cyber.gc.ca");
});
services.TryAddSingleton<HtmlContentSanitizer>();
services.TryAddSingleton<CccsDiagnostics>();
services.TryAddSingleton<CccsHtmlParser>();
services.TryAddSingleton<CccsFeedClient>();
services.AddTransient<CccsConnector>();
return services;
}
}
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Cccs.Internal;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs;
public static class CccsServiceCollectionExtensions
{
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CccsOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0";
clientOptions.Timeout = options.RequestTimeout;
clientOptions.AllowedHosts.Clear();
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
{
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
}
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
clientOptions.AllowedHosts.Add("cyber.gc.ca");
});
services.TryAddSingleton<HtmlContentSanitizer>();
services.TryAddSingleton<CccsDiagnostics>();
services.TryAddSingleton<CccsHtmlParser>();
services.TryAddSingleton<CccsFeedClient>();
services.AddTransient<CccsConnector>();
return services;
}
}

340
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Configuration/CccsOptions.cs
View File

@@ -1,130 +1,130 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Cccs.Configuration;
public sealed class CccsOptions
{
public const string HttpClientName = "concelier.source.cccs";
private readonly List<CccsFeedEndpoint> _feeds = new();
public CccsOptions()
{
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
}
/// <summary>
/// Feed endpoints to poll; configure per language or content category.
/// </summary>
public IList<CccsFeedEndpoint> Feeds => _feeds;
/// <summary>
/// Maximum number of entries to enqueue per fetch cycle.
/// </summary>
public int MaxEntriesPerFetch { get; set; } = 80;
/// <summary>
/// Maximum remembered entries (URI+hash) for deduplication.
/// </summary>
public int MaxKnownEntries { get; set; } = 512;
/// <summary>
/// Timeout applied to feed and taxonomy requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay between successive feed requests to respect upstream throttling.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when fetch fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
public void Validate()
{
if (_feeds.Count == 0)
{
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
}
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var feed in _feeds)
{
feed.Validate();
if (!seenLanguages.Add(feed.Language))
{
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
}
}
if (MaxEntriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
}
if (MaxKnownEntries <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}
public sealed class CccsFeedEndpoint
{
public CccsFeedEndpoint()
{
}
public CccsFeedEndpoint(string language, Uri uri)
{
Language = language;
Uri = uri;
}
public string Language { get; set; } = "en";
public Uri? Uri { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Language))
{
throw new InvalidOperationException("CCCS feed language must be specified.");
}
if (Uri is null || !Uri.IsAbsoluteUri)
{
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
}
}
public Uri BuildTaxonomyUri()
{
if (Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
}
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Cccs.Configuration;
public sealed class CccsOptions
{
public const string HttpClientName = "concelier.source.cccs";
private readonly List<CccsFeedEndpoint> _feeds = new();
public CccsOptions()
{
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
}
/// <summary>
/// Feed endpoints to poll; configure per language or content category.
/// </summary>
public IList<CccsFeedEndpoint> Feeds => _feeds;
/// <summary>
/// Maximum number of entries to enqueue per fetch cycle.
/// </summary>
public int MaxEntriesPerFetch { get; set; } = 80;
/// <summary>
/// Maximum remembered entries (URI+hash) for deduplication.
/// </summary>
public int MaxKnownEntries { get; set; } = 512;
/// <summary>
/// Timeout applied to feed and taxonomy requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay between successive feed requests to respect upstream throttling.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when fetch fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
public void Validate()
{
if (_feeds.Count == 0)
{
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
}
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var feed in _feeds)
{
feed.Validate();
if (!seenLanguages.Add(feed.Language))
{
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
}
}
if (MaxEntriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
}
if (MaxKnownEntries <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}
public sealed class CccsFeedEndpoint
{
public CccsFeedEndpoint()
{
}
public CccsFeedEndpoint(string language, Uri uri)
{
Language = language;
Uri = uri;
}
public string Language { get; set; } = "en";
public Uri? Uri { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Language))
{
throw new InvalidOperationException("CCCS feed language must be specified.");
}
if (Uri is null || !Uri.IsAbsoluteUri)
{
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
}
}
public Uri BuildTaxonomyUri()
{
if (Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
}
var language = Uri.GetQueryParameterValueOrDefault("lang", Language);
var taxonomyBuilder = new UriBuilder(Uri)
{
@@ -135,46 +135,46 @@ public sealed class CccsFeedEndpoint
return taxonomyBuilder.Uri;
}
}
internal static class CccsUriExtensions
{
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
{
if (uri is null)
{
return fallback;
}
var query = uri.Query;
if (string.IsNullOrEmpty(query))
{
return fallback;
}
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex < 0)
{
continue;
}
var left = pair[..separatorIndex].Trim();
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var right = pair[(separatorIndex + 1)..].Trim();
if (right.Length == 0)
{
continue;
}
return Uri.UnescapeDataString(right);
}
return fallback;
}
}
internal static class CccsUriExtensions
{
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
{
if (uri is null)
{
return fallback;
}
var query = uri.Query;
if (string.IsNullOrEmpty(query))
{
return fallback;
}
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex < 0)
{
continue;
}
var left = pair[..separatorIndex].Trim();
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var right = pair[(separatorIndex + 1)..].Trim();
if (right.Length == 0)
{
continue;
}
return Uri.UnescapeDataString(right);
}
return fallback;
}
}

108
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsAdvisoryDto.cs
View File

@@ -1,54 +1,54 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsAdvisoryDto
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string SerialNumber { get; init; } = string.Empty;
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("products")]
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
[JsonPropertyName("references")]
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
}
internal sealed record CccsReferenceDto(
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("label")] string? Label);
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsAdvisoryDto
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string SerialNumber { get; init; } = string.Empty;
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("products")]
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
[JsonPropertyName("references")]
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
}
internal sealed record CccsReferenceDto(
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("label")] string? Label);

290
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsCursor.cs
View File

@@ -1,145 +1,145 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, string> KnownEntryHashes,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingMappings = distinct };
}
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
{
var map = hashes is null || hashes.Count == 0
? EmptyHashes
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
return this with { KnownEntryHashes = map };
}
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var doc = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (KnownEntryHashes.Count > 0)
{
var hashes = new BsonArray();
foreach (var kvp in KnownEntryHashes)
{
hashes.Add(new BsonDocument
{
["uri"] = kvp.Key,
["hash"] = kvp.Value,
});
}
doc["knownEntryHashes"] = hashes;
}
if (LastFetchAt.HasValue)
{
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return doc;
}
public static CccsCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var hashes = ReadHashMap(document);
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
? ParseDateTime(value)
: null;
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidCollection;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
items.Add(guid);
}
}
return items;
}
private static IReadOnlyDictionary<string, string> ReadHashMap(BsonDocument document)
{
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0)
{
return EmptyHashes;
}
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
foreach (var element in array)
{
if (element is not BsonDocument entry)
{
continue;
}
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString))
{
continue;
}
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull
? hashValue.AsString
: string.Empty;
map[uriValue.AsString] = hash;
}
return map;
}
private static DateTimeOffset? ParseDateTime(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, string> KnownEntryHashes,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingMappings = distinct };
}
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
{
var map = hashes is null || hashes.Count == 0
? EmptyHashes
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
return this with { KnownEntryHashes = map };
}
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public DocumentObject ToDocumentObject()
{
var doc = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (KnownEntryHashes.Count > 0)
{
var hashes = new DocumentArray();
foreach (var kvp in KnownEntryHashes)
{
hashes.Add(new DocumentObject
{
["uri"] = kvp.Key,
["hash"] = kvp.Value,
});
}
doc["knownEntryHashes"] = hashes;
}
if (LastFetchAt.HasValue)
{
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return doc;
}
public static CccsCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var hashes = ReadHashMap(document);
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
? ParseDateTime(value)
: null;
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidCollection;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
items.Add(guid);
}
}
return items;
}
private static IReadOnlyDictionary<string, string> ReadHashMap(DocumentObject document)
{
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyHashes;
}
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
foreach (var element in array)
{
if (element is not DocumentObject entry)
{
continue;
}
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsDocumentNull || string.IsNullOrWhiteSpace(uriValue.AsString))
{
continue;
}
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsDocumentNull
? hashValue.AsString
: string.Empty;
map[uriValue.AsString] = hash;
}
return map;
}
private static DateTimeOffset? ParseDateTime(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

116
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsDiagnostics.cs
View File

@@ -1,58 +1,58 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Cccs";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CccsDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchSuccess() => _fetchSuccess.Add(1);
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void ParseQuarantine() => _parseQuarantine.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void Dispose() => _meter.Dispose();
}
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Cccs";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CccsDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchSuccess() => _fetchSuccess.Add(1);
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void ParseQuarantine() => _parseQuarantine.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void Dispose() => _meter.Dispose();
}

292
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedClient.cs
View File

@@ -1,146 +1,146 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Common.Fetch;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsFeedClient
{
private static readonly string[] AcceptHeaders =
{
"application/json",
"application/vnd.api+json;q=0.9",
"text/json;q=0.8",
"application/*+json;q=0.7",
};
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(endpoint);
if (endpoint.Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured.");
}
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = requestTimeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.feedUri"] = endpoint.Uri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
return CccsFeedResult.Empty;
}
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
if (feedResponse is null || feedResponse.Error)
{
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
return CccsFeedResult.Empty;
}
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
return new CccsFeedResult(items, taxonomy, result.LastModified);
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
throw;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
throw;
}
}
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
{
var taxonomyUri = endpoint.BuildTaxonomyUri();
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = timeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
foreach (var item in taxonomyResponse.Response)
{
if (!string.IsNullOrWhiteSpace(item.Title))
{
map[item.Id] = item.Title!;
}
}
return map;
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
}
private static T? Deserialize<T>(byte[] content)
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
}
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Connector.Cccs.Configuration;
using StellaOps.Concelier.Connector.Common.Fetch;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsFeedClient
{
private static readonly string[] AcceptHeaders =
{
"application/json",
"application/vnd.api+json;q=0.9",
"text/json;q=0.8",
"application/*+json;q=0.7",
};
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(endpoint);
if (endpoint.Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured.");
}
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = requestTimeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.feedUri"] = endpoint.Uri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
return CccsFeedResult.Empty;
}
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
if (feedResponse is null || feedResponse.Error)
{
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
return CccsFeedResult.Empty;
}
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
return new CccsFeedResult(items, taxonomy, result.LastModified);
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
throw;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
throw;
}
}
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
{
var taxonomyUri = endpoint.BuildTaxonomyUri();
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = timeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
foreach (var item in taxonomyResponse.Response)
{
if (!string.IsNullOrWhiteSpace(item.Title))
{
map[item.Id] = item.Title!;
}
}
return map;
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
}
private static T? Deserialize<T>(byte[] content)
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
}

202
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedModels.cs
View File

@@ -1,101 +1,101 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed class CccsFeedResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsFeedItem> Response { get; init; } = new();
}
internal sealed class CccsFeedItem
{
[JsonPropertyName("nid")]
public int Nid { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("lang")]
public string? Language { get; init; }
[JsonPropertyName("date_modified")]
public string? DateModified { get; init; }
[JsonPropertyName("date_modified_ts")]
public string? DateModifiedTimestamp { get; init; }
[JsonPropertyName("date_created")]
public string? DateCreated { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("body")]
public string[] Body { get; init; } = Array.Empty<string>();
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("alert_type")]
public JsonElement AlertType { get; init; }
[JsonPropertyName("serial_number")]
public string? SerialNumber { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("moderation_state")]
public string? ModerationState { get; init; }
[JsonPropertyName("external_url")]
public string? ExternalUrl { get; init; }
}
internal sealed class CccsTaxonomyResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsTaxonomyItem> Response { get; init; } = new();
}
internal sealed class CccsTaxonomyItem
{
[JsonPropertyName("id")]
public int Id { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
}
internal sealed record CccsFeedResult(
IReadOnlyList<CccsFeedItem> Items,
IReadOnlyDictionary<int, string> AlertTypes,
DateTimeOffset? LastModifiedUtc)
{
public static CccsFeedResult Empty { get; } = new(
Array.Empty<CccsFeedItem>(),
new Dictionary<int, string>(0),
null);
}
internal static class CccsFeedResultExtensions
{
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
=> new(items, alertTypes, lastModified);
}
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed class CccsFeedResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsFeedItem> Response { get; init; } = new();
}
internal sealed class CccsFeedItem
{
[JsonPropertyName("nid")]
public int Nid { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("lang")]
public string? Language { get; init; }
[JsonPropertyName("date_modified")]
public string? DateModified { get; init; }
[JsonPropertyName("date_modified_ts")]
public string? DateModifiedTimestamp { get; init; }
[JsonPropertyName("date_created")]
public string? DateCreated { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("body")]
public string[] Body { get; init; } = Array.Empty<string>();
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("alert_type")]
public JsonElement AlertType { get; init; }
[JsonPropertyName("serial_number")]
public string? SerialNumber { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("moderation_state")]
public string? ModerationState { get; init; }
[JsonPropertyName("external_url")]
public string? ExternalUrl { get; init; }
}
internal sealed class CccsTaxonomyResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsTaxonomyItem> Response { get; init; } = new();
}
internal sealed class CccsTaxonomyItem
{
[JsonPropertyName("id")]
public int Id { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
}
internal sealed record CccsFeedResult(
IReadOnlyList<CccsFeedItem> Items,
IReadOnlyDictionary<int, string> AlertTypes,
DateTimeOffset? LastModifiedUtc)
{
public static CccsFeedResult Empty { get; } = new(
Array.Empty<CccsFeedItem>(),
new Dictionary<int, string>(0),
null);
}
internal static class CccsFeedResultExtensions
{
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
=> new(items, alertTypes, lastModified);
}

872
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsHtmlParser.cs
View File

@@ -1,353 +1,353 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:：]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:：]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
private static readonly CultureInfo[] EnglishCultures =
{
CultureInfo.GetCultureInfo("en-CA"),
CultureInfo.GetCultureInfo("en-US"),
CultureInfo.InvariantCulture,
};
private static readonly CultureInfo[] FrenchCultures =
{
CultureInfo.GetCultureInfo("fr-CA"),
CultureInfo.GetCultureInfo("fr-FR"),
CultureInfo.InvariantCulture,
};
private static readonly string[] ProductHeadingKeywords =
{
"affected",
"produit",
"produits",
"produits touch",
"produits concern",
"mesures recommand",
};
private static readonly string[] TrackingParameterPrefixes =
{
"utm_",
"mc_",
"mkt_",
"elq",
};
private readonly HtmlContentSanitizer _sanitizer;
private readonly HtmlParser _parser;
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
{
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
_parser = new HtmlParser(new HtmlParserOptions
{
IsScripting = false,
IsKeepingSourceReferences = false,
});
}
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
{
ArgumentNullException.ThrowIfNull(raw);
var baseUri = TryCreateUri(raw.CanonicalUrl);
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
var body = document.Body ?? document.DocumentElement;
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
var contentRoot = body ?? document.DocumentElement;
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
? raw.SerialNumber!.Trim()
: ExtractSerialNumber(document) ?? raw.SourceId;
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
var products = ExtractProducts(contentRoot);
var cveIds = ExtractCveIds(document);
return new CccsAdvisoryDto
{
SourceId = raw.SourceId,
SerialNumber = serialNumber,
Language = raw.Language,
Title = raw.Title,
Summary = CollapseWhitespace(raw.Summary),
CanonicalUrl = raw.CanonicalUrl,
ContentHtml = sanitized,
Published = published,
Modified = raw.Modified ?? published,
AlertType = raw.AlertType,
Subject = raw.Subject,
Products = products,
References = references,
CveIds = cveIds,
};
}
private static Uri? TryCreateUri(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
}
private static string? ExtractSerialNumber(IDocument document)
{
if (document.Body is null)
{
return null;
}
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = SerialRegex.Match(text);
if (match.Success && match.Groups["id"].Success)
{
var value = match.Groups["id"].Value.Trim();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
var bodyText = document.Body.TextContent;
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
return fallback.Success && fallback.Groups["id"].Success
? fallback.Groups["id"].Value.Trim()
: null;
}
private static DateTimeOffset? ExtractDate(IDocument document, string language)
{
if (document.Body is null)
{
return null;
}
var textSegments = new List<string>();
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = DateRegex.Match(text);
if (match.Success && match.Groups["date"].Success)
{
textSegments.Add(match.Groups["date"].Value.Trim());
}
}
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
{
textSegments.Add(document.Body.TextContent);
}
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
foreach (var segment in textSegments)
{
foreach (var culture in cultures)
{
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return new DateTimeOffset(parsed.ToUniversalTime());
}
}
}
return null;
}
private static IReadOnlyList<string> ExtractProducts(IElement? root)
{
if (root is null)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
{
var text = heading.TextContent?.Trim();
if (!IsProductHeading(text))
{
continue;
}
var sibling = heading.NextElementSibling;
while (sibling is not null)
{
if (IsHeading(sibling))
{
break;
}
if (IsListElement(sibling))
{
AppendListItems(sibling, results);
if (results.Count > 0)
{
break;
}
}
else if (IsContentContainer(sibling))
{
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
{
AppendListItems(list, results);
}
if (results.Count > 0)
{
break;
}
}
sibling = sibling.NextElementSibling;
}
if (results.Count > 0)
{
break;
}
}
if (results.Count == 0)
{
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
{
var itemText = CollapseWhitespace(li.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
results.Add(itemText);
}
}
}
return results.Count == 0
? Array.Empty<string>()
: results
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static bool IsProductHeading(string? heading)
{
if (string.IsNullOrWhiteSpace(heading))
{
return false;
}
var lowered = heading.ToLowerInvariant();
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
}
private static bool IsHeading(IElement element)
=> element.LocalName.Length == 2
&& element.LocalName[0] == 'h'
&& char.IsDigit(element.LocalName[1]);
private static bool IsListElement(IElement element)
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
private static bool IsContentContainer(IElement element)
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
{
foreach (var li in listElement.QuerySelectorAll("li"))
{
if (li is null)
{
continue;
}
var clone = li.Clone(true) as IElement;
if (clone is null)
{
continue;
}
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
{
nested.Remove();
}
var itemText = CollapseWhitespace(clone.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
buffer.Add(itemText);
}
}
}
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
{
if (root is null)
{
return Array.Empty<CccsReferenceDto>();
}
var references = new List<CccsReferenceDto>();
foreach (var anchor in root.QuerySelectorAll("a[href]"))
{
var href = anchor.GetAttribute("href");
var normalized = NormalizeReferenceUrl(href, baseUri, language);
if (normalized is null)
{
continue;
}
var label = CollapseWhitespace(anchor.TextContent);
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
}
return references.Count == 0
? Array.Empty<CccsReferenceDto>()
: references
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
{
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:：]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:：]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
private static readonly CultureInfo[] EnglishCultures =
{
CultureInfo.GetCultureInfo("en-CA"),
CultureInfo.GetCultureInfo("en-US"),
CultureInfo.InvariantCulture,
};
private static readonly CultureInfo[] FrenchCultures =
{
CultureInfo.GetCultureInfo("fr-CA"),
CultureInfo.GetCultureInfo("fr-FR"),
CultureInfo.InvariantCulture,
};
private static readonly string[] ProductHeadingKeywords =
{
"affected",
"produit",
"produits",
"produits touch",
"produits concern",
"mesures recommand",
};
private static readonly string[] TrackingParameterPrefixes =
{
"utm_",
"mc_",
"mkt_",
"elq",
};
private readonly HtmlContentSanitizer _sanitizer;
private readonly HtmlParser _parser;
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
{
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
_parser = new HtmlParser(new HtmlParserOptions
{
IsScripting = false,
IsKeepingSourceReferences = false,
});
}
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
{
ArgumentNullException.ThrowIfNull(raw);
var baseUri = TryCreateUri(raw.CanonicalUrl);
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
var body = document.Body ?? document.DocumentElement;
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
var contentRoot = body ?? document.DocumentElement;
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
? raw.SerialNumber!.Trim()
: ExtractSerialNumber(document) ?? raw.SourceId;
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
var products = ExtractProducts(contentRoot);
var cveIds = ExtractCveIds(document);
return new CccsAdvisoryDto
{
SourceId = raw.SourceId,
SerialNumber = serialNumber,
Language = raw.Language,
Title = raw.Title,
Summary = CollapseWhitespace(raw.Summary),
CanonicalUrl = raw.CanonicalUrl,
ContentHtml = sanitized,
Published = published,
Modified = raw.Modified ?? published,
AlertType = raw.AlertType,
Subject = raw.Subject,
Products = products,
References = references,
CveIds = cveIds,
};
}
private static Uri? TryCreateUri(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
}
private static string? ExtractSerialNumber(IDocument document)
{
if (document.Body is null)
{
return null;
}
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = SerialRegex.Match(text);
if (match.Success && match.Groups["id"].Success)
{
var value = match.Groups["id"].Value.Trim();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
var bodyText = document.Body.TextContent;
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
return fallback.Success && fallback.Groups["id"].Success
? fallback.Groups["id"].Value.Trim()
: null;
}
private static DateTimeOffset? ExtractDate(IDocument document, string language)
{
if (document.Body is null)
{
return null;
}
var textSegments = new List<string>();
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = DateRegex.Match(text);
if (match.Success && match.Groups["date"].Success)
{
textSegments.Add(match.Groups["date"].Value.Trim());
}
}
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
{
textSegments.Add(document.Body.TextContent);
}
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
foreach (var segment in textSegments)
{
foreach (var culture in cultures)
{
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return new DateTimeOffset(parsed.ToUniversalTime());
}
}
}
return null;
}
private static IReadOnlyList<string> ExtractProducts(IElement? root)
{
if (root is null)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
{
var text = heading.TextContent?.Trim();
if (!IsProductHeading(text))
{
continue;
}
var sibling = heading.NextElementSibling;
while (sibling is not null)
{
if (IsHeading(sibling))
{
break;
}
if (IsListElement(sibling))
{
AppendListItems(sibling, results);
if (results.Count > 0)
{
break;
}
}
else if (IsContentContainer(sibling))
{
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
{
AppendListItems(list, results);
}
if (results.Count > 0)
{
break;
}
}
sibling = sibling.NextElementSibling;
}
if (results.Count > 0)
{
break;
}
}
if (results.Count == 0)
{
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
{
var itemText = CollapseWhitespace(li.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
results.Add(itemText);
}
}
}
return results.Count == 0
? Array.Empty<string>()
: results
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static bool IsProductHeading(string? heading)
{
if (string.IsNullOrWhiteSpace(heading))
{
return false;
}
var lowered = heading.ToLowerInvariant();
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
}
private static bool IsHeading(IElement element)
=> element.LocalName.Length == 2
&& element.LocalName[0] == 'h'
&& char.IsDigit(element.LocalName[1]);
private static bool IsListElement(IElement element)
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
private static bool IsContentContainer(IElement element)
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
{
foreach (var li in listElement.QuerySelectorAll("li"))
{
if (li is null)
{
continue;
}
var clone = li.Clone(true) as IElement;
if (clone is null)
{
continue;
}
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
{
nested.Remove();
}
var itemText = CollapseWhitespace(clone.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
buffer.Add(itemText);
}
}
}
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
{
if (root is null)
{
return Array.Empty<CccsReferenceDto>();
}
var references = new List<CccsReferenceDto>();
foreach (var anchor in root.QuerySelectorAll("a[href]"))
{
var href = anchor.GetAttribute("href");
var normalized = NormalizeReferenceUrl(href, baseUri, language);
if (normalized is null)
{
continue;
}
var label = CollapseWhitespace(anchor.TextContent);
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
}
return references.Count == 0
? Array.Empty<CccsReferenceDto>()
: references
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
{
if (string.IsNullOrWhiteSpace(href))
{
return null;
@@ -363,89 +363,89 @@ public sealed class CccsHtmlParser
}
}
var builder = new UriBuilder(absolute)
{
Fragment = string.Empty,
};
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
builder.Query = filteredQuery;
return builder.Uri.ToString();
}
private static string FilterTrackingParameters(string query, Uri uri, string language)
{
if (string.IsNullOrWhiteSpace(query))
{
return string.Empty;
}
var trimmed = query.TrimStart('?');
if (string.IsNullOrWhiteSpace(trimmed))
{
return string.Empty;
}
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
var kept = new List<string>();
foreach (var parameter in parameters)
{
var separatorIndex = parameter.IndexOf('=');
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
{
kept.Add($"lang={language}");
continue;
}
kept.Add(parameter);
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
{
kept.Add($"lang={language}");
}
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
}
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
{
if (document.Body is null)
{
return Array.Empty<string>();
}
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
if (matches.Count == 0)
{
return Array.Empty<string>();
}
return matches
.Select(match => match.Value.ToUpperInvariant())
.Distinct(StringComparer.Ordinal)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
}
private static string? CollapseWhitespace(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
return collapsed.Length == 0 ? null : collapsed;
}
}
var builder = new UriBuilder(absolute)
{
Fragment = string.Empty,
};
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
builder.Query = filteredQuery;
return builder.Uri.ToString();
}
private static string FilterTrackingParameters(string query, Uri uri, string language)
{
if (string.IsNullOrWhiteSpace(query))
{
return string.Empty;
}
var trimmed = query.TrimStart('?');
if (string.IsNullOrWhiteSpace(trimmed))
{
return string.Empty;
}
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
var kept = new List<string>();
foreach (var parameter in parameters)
{
var separatorIndex = parameter.IndexOf('=');
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
{
kept.Add($"lang={language}");
continue;
}
kept.Add(parameter);
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
{
kept.Add($"lang={language}");
}
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
}
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
{
if (document.Body is null)
{
return Array.Empty<string>();
}
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
if (matches.Count == 0)
{
return Array.Empty<string>();
}
return matches
.Select(match => match.Value.ToUpperInvariant())
.Distinct(StringComparer.Ordinal)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
}
private static string? CollapseWhitespace(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
return collapsed.Length == 0 ? null : collapsed;
}
}

516
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsMapper.cs
View File

@@ -1,258 +1,258 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Normalization.SemVer;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal static class CccsMapper
{
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"advisory",
dto.AlertType ?? dto.SerialNumber,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory })
};
return new Advisory(
advisoryKey: dto.SerialNumber,
title: dto.Title,
summary: dto.Summary,
language: dto.Language,
published: dto.Published ?? dto.Modified,
modified: dto.Modified ?? dto.Published,
severity: null,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
}
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4)
{
dto.SerialNumber,
};
if (!string.IsNullOrWhiteSpace(dto.SourceId)
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
{
aliases.Add(dto.SourceId);
}
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
dto.CanonicalUrl,
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
"reference",
"cccs",
reference.Label,
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
for (var index = 0; index < dto.Products.Count; index++)
{
var product = dto.Products[index];
if (string.IsNullOrWhiteSpace(product))
{
continue;
}
var identifier = product.Trim();
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var rangeAnchor = $"cccs:{dto.SerialNumber}:{index}";
var versionRanges = BuildVersionRanges(product, rangeAnchor, recordedAt);
var normalizedVersions = BuildNormalizedVersions(versionRanges, rangeAnchor);
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: versionRanges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedVersions));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(string productText, string rangeAnchor, DateTimeOffset recordedAt)
{
var versionText = ExtractFirstVersionToken(productText);
if (string.IsNullOrWhiteSpace(versionText))
{
return Array.Empty<AffectedVersionRange>();
}
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"range",
rangeAnchor,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges });
var vendorExtensions = new Dictionary<string, string>
{
["cccs.version.raw"] = versionText!,
["cccs.anchor"] = rangeAnchor,
};
var semVerResults = SemVerRangeRuleBuilder.Build(versionText!, patchedVersion: null, provenanceNote: rangeAnchor);
if (semVerResults.Count > 0)
{
return semVerResults.Select(result =>
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: result.Primitive.Introduced,
fixedVersion: result.Primitive.Fixed,
lastAffectedVersion: result.Primitive.LastAffected,
rangeExpression: result.Expression ?? versionText!,
provenance: provenance,
primitives: new RangePrimitives(
result.Primitive,
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions)))
.ToArray();
}
var primitives = new RangePrimitives(
new SemVerPrimitive(
Introduced: versionText,
IntroducedInclusive: true,
Fixed: null,
FixedInclusive: false,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null,
ExactValue: versionText),
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions);
return new[]
{
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: versionText,
provenance: provenance,
primitives: primitives),
};
}
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
IReadOnlyList<AffectedVersionRange> ranges,
string rangeAnchor)
{
if (ranges.Count == 0)
{
return Array.Empty<NormalizedVersionRule>();
}
var rules = new List<NormalizedVersionRule>(ranges.Count);
foreach (var range in ranges)
{
var rule = range.ToNormalizedVersionRule(rangeAnchor);
if (rule is not null)
{
rules.Add(rule);
}
}
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
}
private static string? ExtractFirstVersionToken(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var match = Regex.Match(value, @"\d+(?:\.\d+){0,3}(?:[A-Za-z0-9\-_]*)?");
return match.Success ? match.Value : null;
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Normalization.SemVer;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal static class CccsMapper
{
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"advisory",
dto.AlertType ?? dto.SerialNumber,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory })
};
return new Advisory(
advisoryKey: dto.SerialNumber,
title: dto.Title,
summary: dto.Summary,
language: dto.Language,
published: dto.Published ?? dto.Modified,
modified: dto.Modified ?? dto.Published,
severity: null,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
}
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4)
{
dto.SerialNumber,
};
if (!string.IsNullOrWhiteSpace(dto.SourceId)
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
{
aliases.Add(dto.SourceId);
}
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
dto.CanonicalUrl,
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
"reference",
"cccs",
reference.Label,
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
for (var index = 0; index < dto.Products.Count; index++)
{
var product = dto.Products[index];
if (string.IsNullOrWhiteSpace(product))
{
continue;
}
var identifier = product.Trim();
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var rangeAnchor = $"cccs:{dto.SerialNumber}:{index}";
var versionRanges = BuildVersionRanges(product, rangeAnchor, recordedAt);
var normalizedVersions = BuildNormalizedVersions(versionRanges, rangeAnchor);
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: versionRanges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedVersions));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(string productText, string rangeAnchor, DateTimeOffset recordedAt)
{
var versionText = ExtractFirstVersionToken(productText);
if (string.IsNullOrWhiteSpace(versionText))
{
return Array.Empty<AffectedVersionRange>();
}
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"range",
rangeAnchor,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges });
var vendorExtensions = new Dictionary<string, string>
{
["cccs.version.raw"] = versionText!,
["cccs.anchor"] = rangeAnchor,
};
var semVerResults = SemVerRangeRuleBuilder.Build(versionText!, patchedVersion: null, provenanceNote: rangeAnchor);
if (semVerResults.Count > 0)
{
return semVerResults.Select(result =>
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: result.Primitive.Introduced,
fixedVersion: result.Primitive.Fixed,
lastAffectedVersion: result.Primitive.LastAffected,
rangeExpression: result.Expression ?? versionText!,
provenance: provenance,
primitives: new RangePrimitives(
result.Primitive,
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions)))
.ToArray();
}
var primitives = new RangePrimitives(
new SemVerPrimitive(
Introduced: versionText,
IntroducedInclusive: true,
Fixed: null,
FixedInclusive: false,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null,
ExactValue: versionText),
Nevra: null,
Evr: null,
VendorExtensions: vendorExtensions);
return new[]
{
new AffectedVersionRange(
rangeKind: NormalizedVersionSchemes.SemVer,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: versionText,
provenance: provenance,
primitives: primitives),
};
}
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
IReadOnlyList<AffectedVersionRange> ranges,
string rangeAnchor)
{
if (ranges.Count == 0)
{
return Array.Empty<NormalizedVersionRule>();
}
var rules = new List<NormalizedVersionRule>(ranges.Count);
foreach (var range in ranges)
{
var rule = range.ToNormalizedVersionRule(rangeAnchor);
if (rule is not null)
{
rules.Add(rule);
}
}
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
}
private static string? ExtractFirstVersionToken(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var match = Regex.Match(value, @"\d+(?:\.\d+){0,3}(?:[A-Za-z0-9\-_]*)?");
return match.Success ? match.Value : null;
}
}

116
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsRawAdvisoryDocument.cs
View File

@@ -1,58 +1,58 @@
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsRawAdvisoryDocument
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string? SerialNumber { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("externalUrl")]
public string? ExternalUrl { get; init; }
[JsonPropertyName("bodyHtml")]
public string BodyHtml { get; init; } = string.Empty;
[JsonPropertyName("bodySegments")]
public string[] BodySegments { get; init; } = Array.Empty<string>();
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("rawCreated")]
public string? RawDateCreated { get; init; }
[JsonPropertyName("rawModified")]
public string? RawDateModified { get; init; }
}
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Cccs.Internal;
internal sealed record CccsRawAdvisoryDocument
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string? SerialNumber { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("externalUrl")]
public string? ExternalUrl { get; init; }
[JsonPropertyName("bodyHtml")]
public string BodyHtml { get; init; } = string.Empty;
[JsonPropertyName("bodySegments")]
public string[] BodySegments { get; init; } = Array.Empty<string>();
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("rawCreated")]
public string? RawDateCreated { get; init; }
[JsonPropertyName("rawModified")]
public string? RawDateModified { get; init; }
}

44
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Jobs.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Cccs;
internal static class CccsJobKinds
{
public const string Fetch = "source:cccs:fetch";
}
internal sealed class CccsFetchJob : IJob
{
private readonly CccsConnector _connector;
public CccsFetchJob(CccsConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Cccs;
internal static class CccsJobKinds
{
public const string Fetch = "source:cccs:fetch";
}
internal sealed class CccsFetchJob : IJob
{
private readonly CccsConnector _connector;
public CccsFetchJob(CccsConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")]

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs
View File

@@ -6,7 +6,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.CertBund.Configuration;
using StellaOps.Concelier.Connector.CertBund.Internal;
using StellaOps.Concelier.Connector.Common;
@@ -286,7 +286,7 @@ public sealed class CertBundConnector : IFeedConnector
_diagnostics.ParseSuccess(dto.Products.Count, dto.CveIds.Count);
parsedCount++;
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var bson = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", bson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -428,7 +428,7 @@ public sealed class CertBundConnector : IFeedConnector
private Task UpdateCursorAsync(CertBundCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

30
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Internal/CertBundCursor.cs
View File

@@ -1,6 +1,6 @@
using System;
using System.Linq;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
@@ -31,13 +31,13 @@ internal sealed record CertBundCursor(
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
public DocumentObject ToDocumentObject()
{
var document = new BsonDocument
var document = new DocumentObject
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new BsonArray(KnownAdvisories),
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new DocumentArray(KnownAdvisories),
};
if (LastPublished.HasValue)
@@ -53,7 +53,7 @@ internal sealed record CertBundCursor(
return document;
}
public static CertBundCursor FromBson(BsonDocument? document)
public static CertBundCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
@@ -76,9 +76,9 @@ internal sealed record CertBundCursor(
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuids;
}
@@ -95,9 +95,9 @@ internal sealed record CertBundCursor(
return items;
}
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyStrings;
}
@@ -108,11 +108,11 @@ internal sealed record CertBundCursor(
.ToArray();
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcConnector.cs
View File

@@ -9,7 +9,7 @@ using System.Text.Json.Serialization;
using System.Threading;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
@@ -338,7 +338,7 @@ public sealed class CertCcConnector : IFeedConnector
var dto = CertCcNoteParser.Parse(noteBytes, vendorsBytes, vulsBytes, vendorStatusesBytes);
var json = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var payload = StellaOps.Concelier.Bson.BsonDocument.Parse(json);
var payload = StellaOps.Concelier.Documents.DocumentObject.Parse(json);
_diagnostics.ParseSuccess(
dto.Vendors.Count,
@@ -678,7 +678,7 @@ public sealed class CertCcConnector : IFeedConnector
private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
private sealed class NoteDocumentGroup

42
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcConnectorPlugin.cs
View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-cc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertCcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertCcConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-cc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertCcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertCcConnector>();
}
}

100
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs
View File

@@ -1,50 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertCc.Configuration;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-cc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertCcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertCcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertCc.Configuration;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-cc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertCcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertCcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

74
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcServiceCollectionExtensions.cs
View File

@@ -1,37 +1,37 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertCc;
public static class CertCcServiceCollectionExtensions
{
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertCcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
clientOptions.BaseAddress = options.BaseApiUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
});
services.TryAddSingleton<CertCcSummaryPlanner>();
services.TryAddSingleton<CertCcDiagnostics>();
services.AddTransient<CertCcConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertCc;
public static class CertCcServiceCollectionExtensions
{
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertCcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
clientOptions.BaseAddress = options.BaseApiUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
});
services.TryAddSingleton<CertCcSummaryPlanner>();
services.TryAddSingleton<CertCcDiagnostics>();
services.AddTransient<CertCcConnector>();
return services;
}
}

158
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Configuration/CertCcOptions.cs
View File

@@ -1,79 +1,79 @@
using System;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
/// <summary>
/// Connector options governing CERT/CC fetch cadence and API endpoints.
/// </summary>
public sealed class CertCcOptions
{
public const string HttpClientName = "certcc";
/// <summary>
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
/// </summary>
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
/// <summary>
/// Sliding window settings controlling which summary endpoints are requested.
/// </summary>
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
{
WindowSize = TimeSpan.FromDays(30),
Overlap = TimeSpan.FromDays(3),
InitialBackfill = TimeSpan.FromDays(365),
MinimumWindowSize = TimeSpan.FromDays(1),
};
/// <summary>
/// Maximum number of monthly summary endpoints to request in a single plan.
/// </summary>
public int MaxMonthlySummaries { get; set; } = 6;
/// <summary>
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
/// </summary>
public int MaxNotesPerFetch { get; set; } = 25;
/// <summary>
/// Optional delay inserted between successive detail requests to respect upstream throttling.
/// </summary>
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;
public void Validate()
{
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
}
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
}
SummaryWindow ??= new TimeWindowCursorOptions();
SummaryWindow.EnsureValid();
if (MaxMonthlySummaries <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
}
if (MaxNotesPerFetch <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
}
if (DetailRequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
}
}
}
using System;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
/// <summary>
/// Connector options governing CERT/CC fetch cadence and API endpoints.
/// </summary>
public sealed class CertCcOptions
{
public const string HttpClientName = "certcc";
/// <summary>
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
/// </summary>
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
/// <summary>
/// Sliding window settings controlling which summary endpoints are requested.
/// </summary>
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
{
WindowSize = TimeSpan.FromDays(30),
Overlap = TimeSpan.FromDays(3),
InitialBackfill = TimeSpan.FromDays(365),
MinimumWindowSize = TimeSpan.FromDays(1),
};
/// <summary>
/// Maximum number of monthly summary endpoints to request in a single plan.
/// </summary>
public int MaxMonthlySummaries { get; set; } = 6;
/// <summary>
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
/// </summary>
public int MaxNotesPerFetch { get; set; } = 25;
/// <summary>
/// Optional delay inserted between successive detail requests to respect upstream throttling.
/// </summary>
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;
public void Validate()
{
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
}
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
}
SummaryWindow ??= new TimeWindowCursorOptions();
SummaryWindow.EnsureValid();
if (MaxMonthlySummaries <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
}
if (MaxNotesPerFetch <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
}
if (DetailRequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
}
}
}

46
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs
View File

@@ -1,4 +1,4 @@
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
@@ -22,18 +22,18 @@ internal sealed record CertCcCursor(
EmptyGuidArray,
null);
public BsonDocument ToBsonDocument()
public DocumentObject ToDocumentObject()
{
var document = new BsonDocument();
var document = new DocumentObject();
var summary = new BsonDocument();
var summary = new DocumentObject();
SummaryState.WriteTo(summary, "start", "end");
document["summary"] = summary;
document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
document["pendingSummaries"] = new DocumentArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new DocumentArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString()));
if (LastRun.HasValue)
{
@@ -43,7 +43,7 @@ internal sealed record CertCcCursor(
return document;
}
public static CertCcCursor FromBson(BsonDocument? document)
public static CertCcCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
@@ -51,9 +51,9 @@ internal sealed record CertCcCursor(
}
TimeWindowCursorState summaryState = TimeWindowCursorState.Empty;
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument)
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is DocumentObject summaryDocument)
{
summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end");
summaryState = TimeWindowCursorState.FromDocumentObject(summaryDocument, "start", "end");
}
var pendingSummaries = ReadGuidArray(document, "pendingSummaries");
@@ -64,10 +64,10 @@ internal sealed record CertCcCursor(
DateTimeOffset? lastRun = null;
if (document.TryGetValue("lastRun", out var lastRunValue))
{
lastRun = lastRunValue.BsonType switch
lastRun = lastRunValue.DocumentType switch
{
BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
@@ -93,9 +93,9 @@ internal sealed record CertCcCursor(
public CertCcCursor WithLastRun(DateTimeOffset? timestamp)
=> this with { LastRun = timestamp };
private static Guid[] ReadGuidArray(BsonDocument document, string field)
private static Guid[] ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyGuidArray;
}
@@ -112,9 +112,9 @@ internal sealed record CertCcCursor(
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
}
private static string[] ReadStringArray(BsonDocument document, string field)
private static string[] ReadStringArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyStringArray;
}
@@ -124,10 +124,10 @@ internal sealed record CertCcCursor(
{
switch (element)
{
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
case DocumentString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
results.Add(bsonString.AsString.Trim());
break;
case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
case DocumentObject bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
results.Add(inner.AsString.Trim());
break;
}
@@ -142,14 +142,14 @@ internal sealed record CertCcCursor(
.ToArray();
}
private static bool TryReadGuid(BsonValue value, out Guid guid)
private static bool TryReadGuid(DocumentValue value, out Guid guid)
{
if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
if (value is DocumentString bsonString && Guid.TryParse(bsonString.AsString, out guid))
{
return true;
}
if (value is BsonBinaryData binary)
if (value is DocumentBinaryData binary)
{
try
{

428
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcDiagnostics.cs
View File

@@ -1,214 +1,214 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
/// </summary>
public sealed class CertCcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _planWindows;
private readonly Counter<long> _planRequests;
private readonly Histogram<double> _planWindowDays;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchUnchanged;
private readonly Counter<long> _summaryFetchFailures;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchUnchanged;
private readonly Counter<long> _detailFetchMissing;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseVendorCount;
private readonly Histogram<long> _parseStatusCount;
private readonly Histogram<long> _parseVulnerabilityCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffectedPackageCount;
private readonly Histogram<long> _mapNormalizedVersionCount;
public CertCcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_planWindows = _meter.CreateCounter<long>(
name: "certcc.plan.windows",
unit: "windows",
description: "Number of summary planning windows evaluated.");
_planRequests = _meter.CreateCounter<long>(
name: "certcc.plan.requests",
unit: "requests",
description: "Total CERT/CC summary endpoints queued by the planner.");
_planWindowDays = _meter.CreateHistogram<double>(
name: "certcc.plan.window_days",
unit: "day",
description: "Duration of each planning window in days.");
_summaryFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.attempts",
unit: "operations",
description: "Number of VINCE summary fetch attempts.");
_summaryFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.success",
unit: "operations",
description: "Number of VINCE summary fetches persisted to storage.");
_summaryFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.not_modified",
unit: "operations",
description: "Number of VINCE summary fetches returning HTTP 304.");
_summaryFetchFailures = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.failures",
unit: "operations",
description: "Number of VINCE summary fetches that failed after retries.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.attempts",
unit: "operations",
description: "Number of VINCE detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.success",
unit: "operations",
description: "Number of VINCE detail fetches that returned payloads.");
_detailFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.unchanged",
unit: "operations",
description: "Number of VINCE detail fetches returning HTTP 304.");
_detailFetchMissing = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.missing",
unit: "operations",
description: "Number of optional VINCE detail endpoints missing but tolerated.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.failures",
unit: "operations",
description: "Number of VINCE detail fetches that failed after retries.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certcc.parse.success",
unit: "documents",
description: "Number of VINCE note bundles parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certcc.parse.failures",
unit: "documents",
description: "Number of VINCE note bundles that failed to parse.");
_parseVendorCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vendors.count",
unit: "vendors",
description: "Distribution of vendor statements per VINCE note.");
_parseStatusCount = _meter.CreateHistogram<long>(
name: "certcc.parse.statuses.count",
unit: "entries",
description: "Distribution of vendor status entries per VINCE note.");
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vulnerabilities.count",
unit: "entries",
description: "Distribution of vulnerability records per VINCE note.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certcc.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certcc.map.failures",
unit: "advisories",
description: "Number of CERT/CC advisory mapping attempts that failed.");
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
name: "certcc.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per CERT/CC advisory.");
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
name: "certcc.map.normalized_versions.count",
unit: "rules",
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
}
public void PlanEvaluated(TimeWindow window, int requestCount)
{
_planWindows.Add(1);
if (requestCount > 0)
{
_planRequests.Add(requestCount);
}
var duration = window.Duration;
if (duration > TimeSpan.Zero)
{
_planWindowDays.Record(duration.TotalDays);
}
}
public void SummaryFetchAttempt(CertCcSummaryScope scope)
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
public void SummaryFetchSuccess(CertCcSummaryScope scope)
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
public void SummaryFetchFailure(CertCcSummaryScope scope)
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
public void DetailFetchAttempt(string endpoint)
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
public void DetailFetchSuccess(string endpoint)
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
public void DetailFetchUnchanged(string endpoint)
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
public void DetailFetchMissing(string endpoint)
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
public void DetailFetchFailure(string endpoint)
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
{
_parseSuccess.Add(1);
if (vendorCount >= 0)
{
_parseVendorCount.Record(vendorCount);
}
if (statusCount >= 0)
{
_parseStatusCount.Record(statusCount);
}
if (vulnerabilityCount >= 0)
{
_parseVulnerabilityCount.Record(vulnerabilityCount);
}
}
public void ParseFailure()
=> _parseFailures.Add(1);
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
{
_mapSuccess.Add(1);
if (affectedPackageCount >= 0)
{
_mapAffectedPackageCount.Record(affectedPackageCount);
}
if (normalizedVersionCount >= 0)
{
_mapNormalizedVersionCount.Record(normalizedVersionCount);
}
}
public void MapFailure()
=> _mapFailures.Add(1);
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
=> new("scope", scope.ToString().ToLowerInvariant());
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
/// </summary>
public sealed class CertCcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _planWindows;
private readonly Counter<long> _planRequests;
private readonly Histogram<double> _planWindowDays;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchUnchanged;
private readonly Counter<long> _summaryFetchFailures;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchUnchanged;
private readonly Counter<long> _detailFetchMissing;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseVendorCount;
private readonly Histogram<long> _parseStatusCount;
private readonly Histogram<long> _parseVulnerabilityCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffectedPackageCount;
private readonly Histogram<long> _mapNormalizedVersionCount;
public CertCcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_planWindows = _meter.CreateCounter<long>(
name: "certcc.plan.windows",
unit: "windows",
description: "Number of summary planning windows evaluated.");
_planRequests = _meter.CreateCounter<long>(
name: "certcc.plan.requests",
unit: "requests",
description: "Total CERT/CC summary endpoints queued by the planner.");
_planWindowDays = _meter.CreateHistogram<double>(
name: "certcc.plan.window_days",
unit: "day",
description: "Duration of each planning window in days.");
_summaryFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.attempts",
unit: "operations",
description: "Number of VINCE summary fetch attempts.");
_summaryFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.success",
unit: "operations",
description: "Number of VINCE summary fetches persisted to storage.");
_summaryFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.not_modified",
unit: "operations",
description: "Number of VINCE summary fetches returning HTTP 304.");
_summaryFetchFailures = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.failures",
unit: "operations",
description: "Number of VINCE summary fetches that failed after retries.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.attempts",
unit: "operations",
description: "Number of VINCE detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.success",
unit: "operations",
description: "Number of VINCE detail fetches that returned payloads.");
_detailFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.unchanged",
unit: "operations",
description: "Number of VINCE detail fetches returning HTTP 304.");
_detailFetchMissing = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.missing",
unit: "operations",
description: "Number of optional VINCE detail endpoints missing but tolerated.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.failures",
unit: "operations",
description: "Number of VINCE detail fetches that failed after retries.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certcc.parse.success",
unit: "documents",
description: "Number of VINCE note bundles parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certcc.parse.failures",
unit: "documents",
description: "Number of VINCE note bundles that failed to parse.");
_parseVendorCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vendors.count",
unit: "vendors",
description: "Distribution of vendor statements per VINCE note.");
_parseStatusCount = _meter.CreateHistogram<long>(
name: "certcc.parse.statuses.count",
unit: "entries",
description: "Distribution of vendor status entries per VINCE note.");
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vulnerabilities.count",
unit: "entries",
description: "Distribution of vulnerability records per VINCE note.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certcc.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certcc.map.failures",
unit: "advisories",
description: "Number of CERT/CC advisory mapping attempts that failed.");
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
name: "certcc.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per CERT/CC advisory.");
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
name: "certcc.map.normalized_versions.count",
unit: "rules",
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
}
public void PlanEvaluated(TimeWindow window, int requestCount)
{
_planWindows.Add(1);
if (requestCount > 0)
{
_planRequests.Add(requestCount);
}
var duration = window.Duration;
if (duration > TimeSpan.Zero)
{
_planWindowDays.Record(duration.TotalDays);
}
}
public void SummaryFetchAttempt(CertCcSummaryScope scope)
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
public void SummaryFetchSuccess(CertCcSummaryScope scope)
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
public void SummaryFetchFailure(CertCcSummaryScope scope)
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
public void DetailFetchAttempt(string endpoint)
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
public void DetailFetchSuccess(string endpoint)
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
public void DetailFetchUnchanged(string endpoint)
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
public void DetailFetchMissing(string endpoint)
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
public void DetailFetchFailure(string endpoint)
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
{
_parseSuccess.Add(1);
if (vendorCount >= 0)
{
_parseVendorCount.Record(vendorCount);
}
if (statusCount >= 0)
{
_parseStatusCount.Record(statusCount);
}
if (vulnerabilityCount >= 0)
{
_parseVulnerabilityCount.Record(vulnerabilityCount);
}
}
public void ParseFailure()
=> _parseFailures.Add(1);
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
{
_mapSuccess.Add(1);
if (affectedPackageCount >= 0)
{
_mapAffectedPackageCount.Record(affectedPackageCount);
}
if (normalizedVersionCount >= 0)
{
_mapNormalizedVersionCount.Record(normalizedVersionCount);
}
}
public void MapFailure()
=> _mapFailures.Add(1);
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
=> new("scope", scope.ToString().ToLowerInvariant());
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

1214
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcMapper.cs
View File

File diff suppressed because it is too large Load Diff

194
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteDto.cs
View File

@@ -1,97 +1,97 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcNoteDto(
CertCcNoteMetadata Metadata,
IReadOnlyList<CertCcVendorDto> Vendors,
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
{
public static CertCcNoteDto Empty { get; } = new(
CertCcNoteMetadata.Empty,
Array.Empty<CertCcVendorDto>(),
Array.Empty<CertCcVendorStatusDto>(),
Array.Empty<CertCcVulnerabilityDto>());
}
internal sealed record CertCcNoteMetadata(
string? VuId,
string IdNumber,
string Title,
string? Overview,
string? Summary,
DateTimeOffset? Published,
DateTimeOffset? Updated,
DateTimeOffset? Created,
int? Revision,
IReadOnlyList<string> CveIds,
IReadOnlyList<string> PublicUrls,
string? PrimaryUrl)
{
public static CertCcNoteMetadata Empty { get; } = new(
VuId: null,
IdNumber: string.Empty,
Title: string.Empty,
Overview: null,
Summary: null,
Published: null,
Updated: null,
Created: null,
Revision: null,
CveIds: Array.Empty<string>(),
PublicUrls: Array.Empty<string>(),
PrimaryUrl: null);
}
internal sealed record CertCcVendorDto(
string Vendor,
DateTimeOffset? ContactDate,
DateTimeOffset? StatementDate,
DateTimeOffset? Updated,
string? Statement,
string? Addendum,
IReadOnlyList<string> References)
{
public static CertCcVendorDto Empty { get; } = new(
Vendor: string.Empty,
ContactDate: null,
StatementDate: null,
Updated: null,
Statement: null,
Addendum: null,
References: Array.Empty<string>());
}
internal sealed record CertCcVendorStatusDto(
string Vendor,
string CveId,
string Status,
string? Statement,
IReadOnlyList<string> References,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVendorStatusDto Empty { get; } = new(
Vendor: string.Empty,
CveId: string.Empty,
Status: string.Empty,
Statement: null,
References: Array.Empty<string>(),
DateAdded: null,
DateUpdated: null);
}
internal sealed record CertCcVulnerabilityDto(
string CveId,
string? Description,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVulnerabilityDto Empty { get; } = new(
CveId: string.Empty,
Description: null,
DateAdded: null,
DateUpdated: null);
}
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcNoteDto(
CertCcNoteMetadata Metadata,
IReadOnlyList<CertCcVendorDto> Vendors,
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
{
public static CertCcNoteDto Empty { get; } = new(
CertCcNoteMetadata.Empty,
Array.Empty<CertCcVendorDto>(),
Array.Empty<CertCcVendorStatusDto>(),
Array.Empty<CertCcVulnerabilityDto>());
}
internal sealed record CertCcNoteMetadata(
string? VuId,
string IdNumber,
string Title,
string? Overview,
string? Summary,
DateTimeOffset? Published,
DateTimeOffset? Updated,
DateTimeOffset? Created,
int? Revision,
IReadOnlyList<string> CveIds,
IReadOnlyList<string> PublicUrls,
string? PrimaryUrl)
{
public static CertCcNoteMetadata Empty { get; } = new(
VuId: null,
IdNumber: string.Empty,
Title: string.Empty,
Overview: null,
Summary: null,
Published: null,
Updated: null,
Created: null,
Revision: null,
CveIds: Array.Empty<string>(),
PublicUrls: Array.Empty<string>(),
PrimaryUrl: null);
}
internal sealed record CertCcVendorDto(
string Vendor,
DateTimeOffset? ContactDate,
DateTimeOffset? StatementDate,
DateTimeOffset? Updated,
string? Statement,
string? Addendum,
IReadOnlyList<string> References)
{
public static CertCcVendorDto Empty { get; } = new(
Vendor: string.Empty,
ContactDate: null,
StatementDate: null,
Updated: null,
Statement: null,
Addendum: null,
References: Array.Empty<string>());
}
internal sealed record CertCcVendorStatusDto(
string Vendor,
string CveId,
string Status,
string? Statement,
IReadOnlyList<string> References,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVendorStatusDto Empty { get; } = new(
Vendor: string.Empty,
CveId: string.Empty,
Status: string.Empty,
Statement: null,
References: Array.Empty<string>(),
DateAdded: null,
DateUpdated: null);
}
internal sealed record CertCcVulnerabilityDto(
string CveId,
string? Description,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVulnerabilityDto Empty { get; } = new(
CveId: string.Empty,
Description: null,
DateAdded: null,
DateUpdated: null);
}

1078
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteParser.cs
View File

File diff suppressed because it is too large Load Diff

216
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryParser.cs
View File

@@ -1,108 +1,108 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcSummaryParser
{
public static IReadOnlyList<string> ParseNotes(byte[] payload)
{
if (payload is null || payload.Length == 0)
{
return Array.Empty<string>();
}
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
});
var notesElement = document.RootElement.ValueKind switch
{
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
JsonValueKind.Array => document.RootElement,
JsonValueKind.Null or JsonValueKind.Undefined => default,
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
};
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
{
return Array.Empty<string>();
}
var results = new List<string>(notesElement.GetArrayLength());
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var element in notesElement.EnumerateArray())
{
var token = ExtractToken(element);
if (string.IsNullOrWhiteSpace(token))
{
continue;
}
var normalized = token.Trim();
var dedupKey = CreateDedupKey(normalized);
if (seen.Add(dedupKey))
{
results.Add(normalized);
}
}
return results.Count == 0 ? Array.Empty<string>() : results;
}
private static string CreateDedupKey(string token)
{
var digits = string.Concat(token.Where(char.IsDigit));
return digits.Length > 0
? digits
: token.Trim().ToUpperInvariant();
}
private static string? ExtractToken(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number => element.TryGetInt64(out var number)
? number.ToString(CultureInfo.InvariantCulture)
: element.GetRawText(),
JsonValueKind.Object => ExtractFromObject(element),
_ => null,
};
}
private static string? ExtractFromObject(JsonElement element)
{
foreach (var propertyName in PropertyCandidates)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
{
var value = property.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
return null;
}
private static readonly string[] PropertyCandidates =
{
"note",
"notes",
"id",
"idnumber",
"noteId",
"vu",
"vuid",
"vuId",
};
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcSummaryParser
{
public static IReadOnlyList<string> ParseNotes(byte[] payload)
{
if (payload is null || payload.Length == 0)
{
return Array.Empty<string>();
}
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
});
var notesElement = document.RootElement.ValueKind switch
{
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
JsonValueKind.Array => document.RootElement,
JsonValueKind.Null or JsonValueKind.Undefined => default,
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
};
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
{
return Array.Empty<string>();
}
var results = new List<string>(notesElement.GetArrayLength());
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var element in notesElement.EnumerateArray())
{
var token = ExtractToken(element);
if (string.IsNullOrWhiteSpace(token))
{
continue;
}
var normalized = token.Trim();
var dedupKey = CreateDedupKey(normalized);
if (seen.Add(dedupKey))
{
results.Add(normalized);
}
}
return results.Count == 0 ? Array.Empty<string>() : results;
}
private static string CreateDedupKey(string token)
{
var digits = string.Concat(token.Where(char.IsDigit));
return digits.Length > 0
? digits
: token.Trim().ToUpperInvariant();
}
private static string? ExtractToken(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number => element.TryGetInt64(out var number)
? number.ToString(CultureInfo.InvariantCulture)
: element.GetRawText(),
JsonValueKind.Object => ExtractFromObject(element),
_ => null,
};
}
private static string? ExtractFromObject(JsonElement element)
{
foreach (var propertyName in PropertyCandidates)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
{
var value = property.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
return null;
}
private static readonly string[] PropertyCandidates =
{
"note",
"notes",
"id",
"idnumber",
"noteId",
"vu",
"vuid",
"vuId",
};
}

44
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlan.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
public sealed record CertCcSummaryPlan(
TimeWindow Window,
IReadOnlyList<CertCcSummaryRequest> Requests,
TimeWindowCursorState NextState);
public enum CertCcSummaryScope
{
Monthly,
Yearly,
}
public sealed record CertCcSummaryRequest(
Uri Uri,
CertCcSummaryScope Scope,
int Year,
int? Month);
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
public sealed record CertCcSummaryPlan(
TimeWindow Window,
IReadOnlyList<CertCcSummaryRequest> Requests,
TimeWindowCursorState NextState);
public enum CertCcSummaryScope
{
Monthly,
Yearly,
}
public sealed record CertCcSummaryRequest(
Uri Uri,
CertCcSummaryScope Scope,
int Year,
int? Month);

192
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlanner.cs
View File

@@ -1,96 +1,96 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
/// </summary>
public sealed class CertCcSummaryPlanner
{
private readonly CertCcOptions _options;
private readonly TimeProvider _timeProvider;
public CertCcSummaryPlanner(
IOptions<CertCcOptions> options,
TimeProvider? timeProvider = null)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
}
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
{
var now = _timeProvider.GetUtcNow();
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
var months = EnumerateYearMonths(window.Start, window.End)
.Take(_options.MaxMonthlySummaries)
.ToArray();
if (months.Length == 0)
{
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
}
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
foreach (var month in months)
{
requests.Add(new CertCcSummaryRequest(
BuildMonthlyUri(month.Year, month.Month),
CertCcSummaryScope.Monthly,
month.Year,
month.Month));
}
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
{
requests.Add(new CertCcSummaryRequest(
BuildYearlyUri(year),
CertCcSummaryScope.Yearly,
year,
Month: null));
}
return new CertCcSummaryPlan(window, requests, nextState);
}
private Uri BuildMonthlyUri(int year, int month)
{
var path = $"{year:D4}/{month:D2}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private Uri BuildYearlyUri(int year)
{
var path = $"{year:D4}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
{
if (end <= start)
{
yield break;
}
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
{
limit = limit.AddMonths(1);
}
while (cursor < limit)
{
yield return (cursor.Year, cursor.Month);
cursor = cursor.AddMonths(1);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
/// </summary>
public sealed class CertCcSummaryPlanner
{
private readonly CertCcOptions _options;
private readonly TimeProvider _timeProvider;
public CertCcSummaryPlanner(
IOptions<CertCcOptions> options,
TimeProvider? timeProvider = null)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
}
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
{
var now = _timeProvider.GetUtcNow();
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
var months = EnumerateYearMonths(window.Start, window.End)
.Take(_options.MaxMonthlySummaries)
.ToArray();
if (months.Length == 0)
{
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
}
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
foreach (var month in months)
{
requests.Add(new CertCcSummaryRequest(
BuildMonthlyUri(month.Year, month.Month),
CertCcSummaryScope.Monthly,
month.Year,
month.Month));
}
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
{
requests.Add(new CertCcSummaryRequest(
BuildYearlyUri(year),
CertCcSummaryScope.Yearly,
year,
Month: null));
}
return new CertCcSummaryPlan(window, requests, nextState);
}
private Uri BuildMonthlyUri(int year, int month)
{
var path = $"{year:D4}/{month:D2}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private Uri BuildYearlyUri(int year)
{
var path = $"{year:D4}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
{
if (end <= start)
{
yield break;
}
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
{
limit = limit.AddMonths(1);
}
while (cursor < limit)
{
yield return (cursor.Year, cursor.Month);
cursor = cursor.AddMonths(1);
}
}
}

470
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcVendorStatementParser.cs
View File

@@ -1,235 +1,235 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcVendorStatementParser
{
private static readonly string[] PairSeparators =
{
"\t",
" - ",
" – ",
" — ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
{
if (string.IsNullOrWhiteSpace(statement))
{
return Array.Empty<CertCcVendorPatch>();
}
var patches = new List<CertCcVendorPatch>();
var lines = statement
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
if (line.Length == 0)
{
continue;
}
line = TrimBulletPrefix(line);
if (line.Length == 0)
{
continue;
}
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
{
continue;
}
var versions = ExtractVersions(versionSegment);
if (versions.Count == 0)
{
continue;
}
var products = ExtractProducts(productSegment);
if (products.Count == 0)
{
products.Add(string.Empty);
}
if (versions.Count == products.Count)
{
for (var index = 0; index < versions.Count; index++)
{
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
}
continue;
}
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
{
var groupSize = products.Count / versions.Count;
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
{
var start = versionIndex * groupSize;
var end = start + groupSize;
var version = versions[versionIndex];
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
{
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
}
}
continue;
}
var primaryVersion = versions[0];
foreach (var product in products)
{
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
}
}
if (patches.Count == 0)
{
return Array.Empty<CertCcVendorPatch>();
}
return patches
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
.Distinct(CertCcVendorPatch.Comparer)
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
.ToArray();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart(BulletPrefixes).Trim();
return trimmed.Length == 0 ? value.Trim() : trimmed;
}
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
{
foreach (var separator in PairSeparators)
{
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
productSegment = parts[0];
versionSegment = parts[1];
return true;
}
}
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (whitespaceSplit.Length >= 2)
{
productSegment = string.Join(' ', whitespaceSplit[..^1]);
versionSegment = whitespaceSplit[^1];
return true;
}
productSegment = string.Empty;
versionSegment = string.Empty;
return false;
}
private static List<string> ExtractProducts(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var normalized = segment.Replace('\t', ' ').Trim();
var tokens = normalized
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
.Select(static token => token.Trim())
.Where(static token => token.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return tokens;
}
private static List<string> ExtractVersions(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var matches = VersionTokenRegex.Matches(segment);
if (matches.Count == 0)
{
return new List<string>();
}
var versions = new List<string>(matches.Count);
foreach (Match match in matches)
{
if (match.Groups.Count == 0)
{
continue;
}
var value = match.Groups[1].Value.Trim();
if (value.Length == 0)
{
continue;
}
versions.Add(value);
}
return versions
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(32)
.ToList();
}
}
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
{
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
{
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
}
public int GetHashCode(CertCcVendorPatch obj)
{
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
return HashCode.Combine(product, version);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcVendorStatementParser
{
private static readonly string[] PairSeparators =
{
"\t",
" - ",
" – ",
" — ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
{
if (string.IsNullOrWhiteSpace(statement))
{
return Array.Empty<CertCcVendorPatch>();
}
var patches = new List<CertCcVendorPatch>();
var lines = statement
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
if (line.Length == 0)
{
continue;
}
line = TrimBulletPrefix(line);
if (line.Length == 0)
{
continue;
}
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
{
continue;
}
var versions = ExtractVersions(versionSegment);
if (versions.Count == 0)
{
continue;
}
var products = ExtractProducts(productSegment);
if (products.Count == 0)
{
products.Add(string.Empty);
}
if (versions.Count == products.Count)
{
for (var index = 0; index < versions.Count; index++)
{
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
}
continue;
}
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
{
var groupSize = products.Count / versions.Count;
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
{
var start = versionIndex * groupSize;
var end = start + groupSize;
var version = versions[versionIndex];
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
{
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
}
}
continue;
}
var primaryVersion = versions[0];
foreach (var product in products)
{
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
}
}
if (patches.Count == 0)
{
return Array.Empty<CertCcVendorPatch>();
}
return patches
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
.Distinct(CertCcVendorPatch.Comparer)
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
.ToArray();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart(BulletPrefixes).Trim();
return trimmed.Length == 0 ? value.Trim() : trimmed;
}
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
{
foreach (var separator in PairSeparators)
{
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
productSegment = parts[0];
versionSegment = parts[1];
return true;
}
}
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (whitespaceSplit.Length >= 2)
{
productSegment = string.Join(' ', whitespaceSplit[..^1]);
versionSegment = whitespaceSplit[^1];
return true;
}
productSegment = string.Empty;
versionSegment = string.Empty;
return false;
}
private static List<string> ExtractProducts(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var normalized = segment.Replace('\t', ' ').Trim();
var tokens = normalized
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
.Select(static token => token.Trim())
.Where(static token => token.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return tokens;
}
private static List<string> ExtractVersions(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var matches = VersionTokenRegex.Matches(segment);
if (matches.Count == 0)
{
return new List<string>();
}
var versions = new List<string>(matches.Count);
foreach (Match match in matches)
{
if (match.Groups.Count == 0)
{
continue;
}
var value = match.Groups[1].Value.Trim();
if (value.Length == 0)
{
continue;
}
versions.Add(value);
}
return versions
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(32)
.ToList();
}
}
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
{
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
{
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
}
public int GetHashCode(CertCcVendorPatch obj)
{
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
return HashCode.Combine(product, version);
}
}
}

44
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Jobs.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertCc;
internal static class CertCcJobKinds
{
public const string Fetch = "source:cert-cc:fetch";
}
internal sealed class CertCcFetchJob : IJob
{
private readonly CertCcConnector _connector;
public CertCcFetchJob(CertCcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertCc;
internal static class CertCcJobKinds
{
public const string Fetch = "source:cert-cc:fetch";
}
internal sealed class CertCcFetchJob : IJob
{
private readonly CertCcConnector _connector;
public CertCcFetchJob(CertCcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/CertFrConnector.cs
View File

@@ -4,7 +4,7 @@ using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.CertFr.Configuration;
using StellaOps.Concelier.Connector.CertFr.Internal;
using StellaOps.Concelier.Connector.Common;
@@ -236,7 +236,7 @@ public sealed class CertFrConnector : IFeedConnector
}
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = BsonDocument.Parse(json);
var payload = DocumentObject.Parse(json);
var validatedAt = _timeProvider.GetUtcNow();
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
@@ -332,6 +332,6 @@ public sealed class CertFrConnector : IFeedConnector
private async Task UpdateCursorAsync(CertFrCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
}

42
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/CertFrConnectorPlugin.cs
View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-fr";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertFrConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertFrConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-fr";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertFrConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertFrConnector>();
}
}

108
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/CertFrDependencyInjectionRoutine.cs
View File

@@ -1,54 +1,54 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-fr";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertFrConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertFrFetchJob>();
services.AddTransient<CertFrParseJob>();
services.AddTransient<CertFrMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob));
EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob));
EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-fr";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertFrConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertFrFetchJob>();
services.AddTransient<CertFrParseJob>();
services.AddTransient<CertFrMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob));
EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob));
EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

72
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/CertFrServiceCollectionExtensions.cs
View File

@@ -1,36 +1,36 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
using StellaOps.Concelier.Connector.CertFr.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertFr;
public static class CertFrServiceCollectionExtensions
{
public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action<CertFrOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertFrOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value;
clientOptions.BaseAddress = options.FeedUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
});
services.TryAddSingleton<CertFrFeedClient>();
services.AddTransient<CertFrConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
using StellaOps.Concelier.Connector.CertFr.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertFr;
public static class CertFrServiceCollectionExtensions
{
public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action<CertFrOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertFrOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value;
clientOptions.BaseAddress = options.FeedUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
});
services.TryAddSingleton<CertFrFeedClient>();
services.AddTransient<CertFrConnector>();
return services;
}
}

92
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs
View File

@@ -1,46 +1,46 @@
using System;
namespace StellaOps.Concelier.Connector.CertFr.Configuration;
public sealed class CertFrOptions
{
public const string HttpClientName = "cert-fr";
public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
public int MaxItemsPerFetch { get; set; } = 100;
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("InitialBackfill must be a positive duration.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (MaxItemsPerFetch <= 0)
{
throw new InvalidOperationException("MaxItemsPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}
using System;
namespace StellaOps.Concelier.Connector.CertFr.Configuration;
public sealed class CertFrOptions
{
public const string HttpClientName = "cert-fr";
public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
public int MaxItemsPerFetch { get; set; } = 100;
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("InitialBackfill must be a positive duration.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (MaxItemsPerFetch <= 0)
{
throw new InvalidOperationException("MaxItemsPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}

176
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrCursor.cs
View File

@@ -1,88 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertFrCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertFrCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var value)
? ParseDate(value)
: null;
return new CertFrCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertFrCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array)
{
return Array.Empty<Guid>();
}
var result = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
result.Add(guid);
}
}
return result;
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertFrCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertFrCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var value)
? ParseDate(value)
: null;
return new CertFrCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertFrCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var raw) || raw is not DocumentArray array)
{
return Array.Empty<Guid>();
}
var result = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
result.Add(guid);
}
}
return result;
}
}

154
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDocumentMetadata.cs
View File

@@ -1,77 +1,77 @@
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDocumentMetadata(
string AdvisoryId,
string Title,
DateTimeOffset Published,
Uri DetailUri,
string? Summary)
{
private const string AdvisoryIdKey = "certfr.advisoryId";
private const string TitleKey = "certfr.title";
private const string PublishedKey = "certfr.published";
private const string SummaryKey = "certfr.summary";
public static CertFrDocumentMetadata FromDocument(DocumentRecord document)
{
ArgumentNullException.ThrowIfNull(document);
if (document.Metadata is null)
{
throw new InvalidOperationException("Cert-FR document metadata is missing.");
}
var metadata = document.Metadata;
if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId))
{
throw new InvalidOperationException("Cert-FR advisory id metadata missing.");
}
if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title))
{
throw new InvalidOperationException("Cert-FR title metadata missing.");
}
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
{
throw new InvalidOperationException("Cert-FR published metadata invalid.");
}
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
{
throw new InvalidOperationException("Cert-FR document URI invalid.");
}
metadata.TryGetValue(SummaryKey, out var summary);
return new CertFrDocumentMetadata(
advisoryId.Trim(),
title.Trim(),
published.ToUniversalTime(),
detailUri,
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
}
public static IReadOnlyDictionary<string, string> CreateMetadata(CertFrFeedItem item)
{
ArgumentNullException.ThrowIfNull(item);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
[AdvisoryIdKey] = item.AdvisoryId,
[TitleKey] = item.Title ?? item.AdvisoryId,
[PublishedKey] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Summary))
{
metadata[SummaryKey] = item.Summary!;
}
return metadata;
}
}
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDocumentMetadata(
string AdvisoryId,
string Title,
DateTimeOffset Published,
Uri DetailUri,
string? Summary)
{
private const string AdvisoryIdKey = "certfr.advisoryId";
private const string TitleKey = "certfr.title";
private const string PublishedKey = "certfr.published";
private const string SummaryKey = "certfr.summary";
public static CertFrDocumentMetadata FromDocument(DocumentRecord document)
{
ArgumentNullException.ThrowIfNull(document);
if (document.Metadata is null)
{
throw new InvalidOperationException("Cert-FR document metadata is missing.");
}
var metadata = document.Metadata;
if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId))
{
throw new InvalidOperationException("Cert-FR advisory id metadata missing.");
}
if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title))
{
throw new InvalidOperationException("Cert-FR title metadata missing.");
}
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
{
throw new InvalidOperationException("Cert-FR published metadata invalid.");
}
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
{
throw new InvalidOperationException("Cert-FR document URI invalid.");
}
metadata.TryGetValue(SummaryKey, out var summary);
return new CertFrDocumentMetadata(
advisoryId.Trim(),
title.Trim(),
published.ToUniversalTime(),
detailUri,
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
}
public static IReadOnlyDictionary<string, string> CreateMetadata(CertFrFeedItem item)
{
ArgumentNullException.ThrowIfNull(item);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
[AdvisoryIdKey] = item.AdvisoryId,
[TitleKey] = item.Title ?? item.AdvisoryId,
[PublishedKey] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Summary))
{
metadata[SummaryKey] = item.Summary!;
}
return metadata;
}
}

28
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDto.cs
View File

@@ -1,14 +1,14 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDto(
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("detailUrl")] string DetailUrl,
[property: JsonPropertyName("published")] DateTimeOffset Published,
[property: JsonPropertyName("summary")] string? Summary,
[property: JsonPropertyName("content")] string Content,
[property: JsonPropertyName("references")] IReadOnlyList<string> References);
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal sealed record CertFrDto(
[property: JsonPropertyName("advisoryId")] string AdvisoryId,
[property: JsonPropertyName("title")] string Title,
[property: JsonPropertyName("detailUrl")] string DetailUrl,
[property: JsonPropertyName("published")] DateTimeOffset Published,
[property: JsonPropertyName("summary")] string? Summary,
[property: JsonPropertyName("content")] string Content,
[property: JsonPropertyName("references")] IReadOnlyList<string> References);

218
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedClient.cs
View File

@@ -1,109 +1,109 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed class CertFrFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertFrOptions _options;
private readonly ILogger<CertFrFeedClient> _logger;
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertFrFeedItem>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertFrFeedItem>();
var now = DateTimeOffset.UtcNow;
foreach (var itemElement in document.Descendants("item"))
{
var link = itemElement.Element("link")?.Value;
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
{
continue;
}
var title = itemElement.Element("title")?.Value?.Trim();
var summary = itemElement.Element("description")?.Value?.Trim();
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
if (published < windowStart)
{
continue;
}
if (published > windowEnd)
{
published = windowEnd;
}
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
}
return items
.OrderBy(item => item.Published)
.Take(_options.MaxItemsPerFetch)
.ToArray();
}
private static DateTimeOffset? ParsePublished(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed;
}
return null;
}
private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri)
{
var guid = itemElement.Element("guid")?.Value;
if (!string.IsNullOrWhiteSpace(guid))
{
return guid.Trim();
}
var segments = detailUri.Segments;
if (segments.Length > 0)
{
var slug = segments[^1].Trim('/');
if (!string.IsNullOrWhiteSpace(slug))
{
return slug;
}
}
return detailUri.AbsoluteUri;
}
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertFr.Configuration;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed class CertFrFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertFrOptions _options;
private readonly ILogger<CertFrFeedClient> _logger;
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertFrFeedItem>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertFrFeedItem>();
var now = DateTimeOffset.UtcNow;
foreach (var itemElement in document.Descendants("item"))
{
var link = itemElement.Element("link")?.Value;
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
{
continue;
}
var title = itemElement.Element("title")?.Value?.Trim();
var summary = itemElement.Element("description")?.Value?.Trim();
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
if (published < windowStart)
{
continue;
}
if (published > windowEnd)
{
published = windowEnd;
}
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
}
return items
.OrderBy(item => item.Published)
.Take(_options.MaxItemsPerFetch)
.ToArray();
}
private static DateTimeOffset? ParsePublished(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed;
}
return null;
}
private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri)
{
var guid = itemElement.Element("guid")?.Value;
if (!string.IsNullOrWhiteSpace(guid))
{
return guid.Trim();
}
var segments = detailUri.Segments;
if (segments.Length > 0)
{
var slug = segments[^1].Trim('/');
if (!string.IsNullOrWhiteSpace(slug))
{
return slug;
}
}
return detailUri.AbsoluteUri;
}
}

20
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedItem.cs
View File

@@ -1,10 +1,10 @@
using System;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed record CertFrFeedItem(
string AdvisoryId,
Uri DetailUri,
DateTimeOffset Published,
string? Title,
string? Summary);
using System;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed record CertFrFeedItem(
string AdvisoryId,
Uri DetailUri,
DateTimeOffset Published,
string? Title,
string? Summary);

76
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrMapper.cs
View File

@@ -1,25 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrMapper
{
public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
var advisoryKey = $"cert-fr/{dto.AdvisoryId}";
var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime());
var aliases = new List<string>
{
$"CERT-FR:{dto.AdvisoryId}",
};
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrMapper
{
public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentException.ThrowIfNullOrEmpty(sourceName);
var advisoryKey = $"cert-fr/{dto.AdvisoryId}";
var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime());
var aliases = new List<string>
{
$"CERT-FR:{dto.AdvisoryId}",
};
var references = BuildReferences(dto, provenance).ToArray();
var affectedPackages = BuildAffectedPackages(dto, provenance).ToArray();
@@ -45,22 +45,22 @@ internal static class CertFrMapper
var comparer = StringComparer.OrdinalIgnoreCase;
var entries = new List<(AdvisoryReference Reference, int Priority)>
{
(new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0),
};
foreach (var url in dto.References)
{
entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1));
}
return entries
.GroupBy(tuple => tuple.Reference.Url, comparer)
.Select(group => group
.OrderBy(t => t.Priority)
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
.ThenBy(t => t.Reference.Url, comparer)
.First())
.OrderBy(t => t.Priority)
(new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0),
};
foreach (var url in dto.References)
{
entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1));
}
return entries
.GroupBy(tuple => tuple.Reference.Url, comparer)
.Select(group => group
.OrderBy(t => t.Priority)
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
.ThenBy(t => t.Reference.Url, comparer)
.First())
.OrderBy(t => t.Priority)
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
.ThenBy(t => t.Reference.Url, comparer)
.Select(t => t.Reference);

160
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrParser.cs
View File

@@ -1,80 +1,80 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrParser
{
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata)
{
ArgumentException.ThrowIfNullOrEmpty(html);
ArgumentNullException.ThrowIfNull(metadata);
var sanitized = SanitizeHtml(html);
var summary = BuildSummary(metadata.Summary, sanitized);
var references = ExtractReferences(html);
return new CertFrDto(
metadata.AdvisoryId,
metadata.Title,
metadata.DetailUri.ToString(),
metadata.Published,
summary,
sanitized,
references);
}
private static string SanitizeHtml(string html)
{
var withoutScripts = ScriptRegex.Replace(html, string.Empty);
var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty);
var withoutTags = TagRegex.Replace(withoutStyles, " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
return WhitespaceRegex.Replace(decoded, " ").Trim();
}
private static string? BuildSummary(string? metadataSummary, string content)
{
if (!string.IsNullOrWhiteSpace(metadataSummary))
{
return metadataSummary.Trim();
}
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (sentences.Length > 0)
{
return sentences[0].Trim();
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static IReadOnlyList<string> ExtractReferences(string html)
{
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in AnchorRegex.Matches(html))
{
if (match.Success)
{
references.Add(match.Groups["url"].Value.Trim());
}
}
return references.Count == 0
? Array.Empty<string>()
: references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray();
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrParser
{
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata)
{
ArgumentException.ThrowIfNullOrEmpty(html);
ArgumentNullException.ThrowIfNull(metadata);
var sanitized = SanitizeHtml(html);
var summary = BuildSummary(metadata.Summary, sanitized);
var references = ExtractReferences(html);
return new CertFrDto(
metadata.AdvisoryId,
metadata.Title,
metadata.DetailUri.ToString(),
metadata.Published,
summary,
sanitized,
references);
}
private static string SanitizeHtml(string html)
{
var withoutScripts = ScriptRegex.Replace(html, string.Empty);
var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty);
var withoutTags = TagRegex.Replace(withoutStyles, " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
return WhitespaceRegex.Replace(decoded, " ").Trim();
}
private static string? BuildSummary(string? metadataSummary, string content)
{
if (!string.IsNullOrWhiteSpace(metadataSummary))
{
return metadataSummary.Trim();
}
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (sentences.Length > 0)
{
return sentences[0].Trim();
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static IReadOnlyList<string> ExtractReferences(string html)
{
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in AnchorRegex.Matches(html))
{
if (match.Success)
{
references.Add(match.Groups["url"].Value.Trim());
}
}
return references.Count == 0
? Array.Empty<string>()
: references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray();
}
}

92
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Jobs.cs
View File

@@ -1,46 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertFr;
internal static class CertFrJobKinds
{
public const string Fetch = "source:cert-fr:fetch";
public const string Parse = "source:cert-fr:parse";
public const string Map = "source:cert-fr:map";
}
internal sealed class CertFrFetchJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrFetchJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertFrParseJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrParseJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertFrMapJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrMapJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertFr;
internal static class CertFrJobKinds
{
public const string Fetch = "source:cert-fr:fetch";
public const string Parse = "source:cert-fr:parse";
public const string Map = "source:cert-fr:map";
}
internal sealed class CertFrFetchJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrFetchJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertFrParseJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrParseJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertFrMapJob : IJob
{
private readonly CertFrConnector _connector;
public CertFrMapJob(CertFrConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

10
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/CertInConnector.cs
View File

@@ -6,7 +6,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
@@ -226,7 +226,7 @@ public sealed class CertInConnector : IFeedConnector
}
var dto = CertInDetailParser.Parse(listing, rawBytes);
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var payload = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certin.v1", payload, _timeProvider.GetUtcNow());
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
@@ -271,9 +271,9 @@ public sealed class CertInConnector : IFeedConnector
continue;
}
var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Bson.IO.JsonWriterSettings
var dtoJson = dtoRecord.Payload.ToJson(new StellaOps.Concelier.Documents.IO.JsonWriterSettings
{
OutputMode = StellaOps.Concelier.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
OutputMode = StellaOps.Concelier.Documents.IO.JsonOutputMode.RelaxedExtendedJson,
});
CertInAdvisoryDto dto;
@@ -423,7 +423,7 @@ public sealed class CertInConnector : IFeedConnector
private Task UpdateCursorAsync(CertInCursor cursor, CancellationToken cancellationToken)
{
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken);
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), _timeProvider.GetUtcNow(), cancellationToken);
}
private static bool TryDeserializeListing(IReadOnlyDictionary<string, string>? metadata, out CertInListingItem listing)

38
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/CertInConnectorPlugin.cs
View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-in";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<CertInConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-in";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<CertInConnector>(services);
}
}

108
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/CertInDependencyInjectionRoutine.cs
View File

@@ -1,54 +1,54 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-in";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertInConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertInFetchJob>();
services.AddTransient<CertInParseJob>();
services.AddTransient<CertInMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob));
EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob));
EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-in";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertInConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertInFetchJob>();
services.AddTransient<CertInParseJob>();
services.AddTransient<CertInMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob));
EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob));
EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

74
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/CertInServiceCollectionExtensions.cs
View File

@@ -1,37 +1,37 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertIn;
public static class CertInServiceCollectionExtensions
{
public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action<CertInOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertInOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value;
clientOptions.BaseAddress = options.AlertsEndpoint;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host);
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddTransient<CertInClient>();
services.AddTransient<CertInConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertIn;
public static class CertInServiceCollectionExtensions
{
public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action<CertInOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertInOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value;
clientOptions.BaseAddress = options.AlertsEndpoint;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host);
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddTransient<CertInClient>();
services.AddTransient<CertInConnector>();
return services;
}
}

136
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Configuration/CertInOptions.cs
View File

@@ -1,68 +1,68 @@
using System;
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Concelier.Connector.CertIn.Configuration;
public sealed class CertInOptions
{
public static string HttpClientName => "source.certin";
/// <summary>
/// Endpoint returning a paginated list of recent advisories.
/// </summary>
public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute);
/// <summary>
/// Size of the rolling fetch window.
/// </summary>
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
/// <summary>
/// Overlap applied to subsequent windows.
/// </summary>
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
/// <summary>
/// Maximum pages fetched per cycle.
/// </summary>
public int MaxPagesPerFetch { get; set; } = 5;
/// <summary>
/// Delay between successive HTTP requests.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
[MemberNotNull(nameof(AlertsEndpoint))]
public void Validate()
{
if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("AlertsEndpoint must be an absolute URI.");
}
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("WindowSize must be greater than zero.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (WindowOverlap >= WindowSize)
{
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}
using System;
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Concelier.Connector.CertIn.Configuration;
public sealed class CertInOptions
{
public static string HttpClientName => "source.certin";
/// <summary>
/// Endpoint returning a paginated list of recent advisories.
/// </summary>
public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute);
/// <summary>
/// Size of the rolling fetch window.
/// </summary>
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
/// <summary>
/// Overlap applied to subsequent windows.
/// </summary>
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
/// <summary>
/// Maximum pages fetched per cycle.
/// </summary>
public int MaxPagesPerFetch { get; set; } = 5;
/// <summary>
/// Delay between successive HTTP requests.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
[MemberNotNull(nameof(AlertsEndpoint))]
public void Validate()
{
if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("AlertsEndpoint must be an absolute URI.");
}
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("WindowSize must be greater than zero.");
}
if (WindowOverlap < TimeSpan.Zero)
{
throw new InvalidOperationException("WindowOverlap cannot be negative.");
}
if (WindowOverlap >= WindowSize)
{
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("RequestDelay cannot be negative.");
}
}
}

32
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInAdvisoryDto.cs
View File

@@ -1,16 +1,16 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInAdvisoryDto(
string AdvisoryId,
string Title,
string Link,
DateTimeOffset Published,
string? Summary,
string Content,
string? Severity,
ImmutableArray<string> CveIds,
ImmutableArray<string> VendorNames,
ImmutableArray<string> ReferenceLinks);
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInAdvisoryDto(
string AdvisoryId,
string Title,
string Link,
DateTimeOffset Published,
string? Summary,
string Content,
string? Severity,
ImmutableArray<string> CveIds,
ImmutableArray<string> VendorNames,
ImmutableArray<string> ReferenceLinks);

258
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInClient.cs
View File

@@ -1,129 +1,129 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed class CertInClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertInOptions _options;
private readonly ILogger<CertInClient> _logger;
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
{
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
{
continue;
}
items.Add(item);
}
return items;
}
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
{
item = null!;
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
{
return false;
}
var advisoryId = idElement.GetString();
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
? titleElement.GetString()
: advisoryId;
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
{
return false;
}
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
{
return false;
}
DateTimeOffset published;
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
{
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
{
return false;
}
}
else
{
return false;
}
string? summary = null;
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
{
summary = summaryElement.GetString();
}
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
return true;
}
private static Uri BuildPageUri(Uri baseUri, int page)
{
if (page <= 1)
{
return baseUri;
}
var builder = new UriBuilder(baseUri);
var trimmed = builder.Query.TrimStart('?');
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(trimmed)
? pageSegment
: $"{trimmed}&{pageSegment}";
return builder.Uri;
}
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed class CertInClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertInOptions _options;
private readonly ILogger<CertInClient> _logger;
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
{
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
{
continue;
}
items.Add(item);
}
return items;
}
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
{
item = null!;
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
{
return false;
}
var advisoryId = idElement.GetString();
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
? titleElement.GetString()
: advisoryId;
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
{
return false;
}
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
{
return false;
}
DateTimeOffset published;
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
{
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
{
return false;
}
}
else
{
return false;
}
string? summary = null;
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
{
summary = summaryElement.GetString();
}
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
return true;
}
private static Uri BuildPageUri(Uri baseUri, int page)
{
if (page <= 1)
{
return baseUri;
}
var builder = new UriBuilder(baseUri);
var trimmed = builder.Query.TrimStart('?');
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(trimmed)
? pageSegment
: $"{trimmed}&{pageSegment}";
return builder.Uri;
}
}

176
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInCursor.cs
View File

@@ -1,88 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertInCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
? ParseDate(dateValue)
: null;
return new CertInCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertInCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
? ParseDate(dateValue)
: null;
return new CertInCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}

374
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInDetailParser.cs
View File

@@ -1,187 +1,187 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal static class CertInDetailParser
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
{
ArgumentNullException.ThrowIfNull(listing);
var html = Encoding.UTF8.GetString(rawHtml);
var content = HtmlToPlainText(html);
var summary = listing.Summary ?? ExtractSummary(content);
var severity = ExtractSeverity(content);
var cves = ExtractCves(listing.Title, summary, content);
var vendors = ExtractVendors(summary, content);
var references = ExtractLinks(html);
return new CertInAdvisoryDto(
listing.AdvisoryId,
listing.Title,
listing.DetailUri.ToString(),
listing.Published,
summary,
content,
severity,
cves,
vendors,
references);
}
private static string HtmlToPlainText(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
return string.IsNullOrWhiteSpace(decoded)
? string.Empty
: Regex.Replace(decoded, "\\s+", " ").Trim();
}
private static string? ExtractSummary(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentenceTerminators = new[] { ".", "!", "?" };
foreach (var terminator in sentenceTerminators)
{
var index = content.IndexOf(terminator, StringComparison.Ordinal);
if (index > 0)
{
return content[..(index + terminator.Length)].Trim();
}
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static string? ExtractSeverity(string content)
{
var match = SeverityRegex.Match(content);
if (match.Success)
{
return match.Groups["value"].Value.Trim().ToLowerInvariant();
}
return null;
}
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Capture(string? text)
{
if (string.IsNullOrWhiteSpace(text))
{
return;
}
foreach (Match match in CveRegex.Matches(text))
{
if (match.Success)
{
set.Add(match.Value.ToUpperInvariant());
}
}
}
Capture(title);
Capture(summary);
Capture(content);
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
{
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Add(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var cleaned = value
.Replace("’", "'", StringComparison.Ordinal)
.Trim();
if (cleaned.Length > 200)
{
cleaned = cleaned[..200];
}
if (!string.IsNullOrWhiteSpace(cleaned))
{
vendors.Add(cleaned);
}
}
if (!string.IsNullOrWhiteSpace(summary))
{
foreach (Match match in VendorRegex.Matches(summary))
{
Add(match.Groups["value"].Value);
}
}
foreach (Match match in VendorRegex.Matches(content))
{
Add(match.Groups["value"].Value);
}
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
{
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
Add(fallback);
}
return vendors.Count == 0
? ImmutableArray<string>.Empty
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractLinks(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ImmutableArray<string>.Empty;
}
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in LinkRegex.Matches(html))
{
if (match.Success)
{
links.Add(match.Groups[1].Value);
}
}
return links.Count == 0
? ImmutableArray<string>.Empty
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
}
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal static class CertInDetailParser
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
{
ArgumentNullException.ThrowIfNull(listing);
var html = Encoding.UTF8.GetString(rawHtml);
var content = HtmlToPlainText(html);
var summary = listing.Summary ?? ExtractSummary(content);
var severity = ExtractSeverity(content);
var cves = ExtractCves(listing.Title, summary, content);
var vendors = ExtractVendors(summary, content);
var references = ExtractLinks(html);
return new CertInAdvisoryDto(
listing.AdvisoryId,
listing.Title,
listing.DetailUri.ToString(),
listing.Published,
summary,
content,
severity,
cves,
vendors,
references);
}
private static string HtmlToPlainText(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
return string.IsNullOrWhiteSpace(decoded)
? string.Empty
: Regex.Replace(decoded, "\\s+", " ").Trim();
}
private static string? ExtractSummary(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentenceTerminators = new[] { ".", "!", "?" };
foreach (var terminator in sentenceTerminators)
{
var index = content.IndexOf(terminator, StringComparison.Ordinal);
if (index > 0)
{
return content[..(index + terminator.Length)].Trim();
}
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static string? ExtractSeverity(string content)
{
var match = SeverityRegex.Match(content);
if (match.Success)
{
return match.Groups["value"].Value.Trim().ToLowerInvariant();
}
return null;
}
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Capture(string? text)
{
if (string.IsNullOrWhiteSpace(text))
{
return;
}
foreach (Match match in CveRegex.Matches(text))
{
if (match.Success)
{
set.Add(match.Value.ToUpperInvariant());
}
}
}
Capture(title);
Capture(summary);
Capture(content);
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
{
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Add(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var cleaned = value
.Replace("’", "'", StringComparison.Ordinal)
.Trim();
if (cleaned.Length > 200)
{
cleaned = cleaned[..200];
}
if (!string.IsNullOrWhiteSpace(cleaned))
{
vendors.Add(cleaned);
}
}
if (!string.IsNullOrWhiteSpace(summary))
{
foreach (Match match in VendorRegex.Matches(summary))
{
Add(match.Groups["value"].Value);
}
}
foreach (Match match in VendorRegex.Matches(content))
{
Add(match.Groups["value"].Value);
}
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
{
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
Add(fallback);
}
return vendors.Count == 0
? ImmutableArray<string>.Empty
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractLinks(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ImmutableArray<string>.Empty;
}
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in LinkRegex.Matches(html))
{
if (match.Success)
{
links.Add(match.Groups[1].Value);
}
}
return links.Count == 0
? ImmutableArray<string>.Empty
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
}

20
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInListingItem.cs
View File

@@ -1,10 +1,10 @@
using System;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed record CertInListingItem(
string AdvisoryId,
string Title,
Uri DetailUri,
DateTimeOffset Published,
string? Summary);
using System;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed record CertInListingItem(
string AdvisoryId,
string Title,
Uri DetailUri,
DateTimeOffset Published,
string? Summary);

92
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Jobs.cs
View File

@@ -1,46 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertIn;
internal static class CertInJobKinds
{
public const string Fetch = "source:cert-in:fetch";
public const string Parse = "source:cert-in:parse";
public const string Map = "source:cert-in:map";
}
internal sealed class CertInFetchJob : IJob
{
private readonly CertInConnector _connector;
public CertInFetchJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertInParseJob : IJob
{
private readonly CertInConnector _connector;
public CertInParseJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertInMapJob : IJob
{
private readonly CertInConnector _connector;
public CertInMapJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertIn;
internal static class CertInJobKinds
{
public const string Fetch = "source:cert-in:fetch";
public const string Parse = "source:cert-in:parse";
public const string Map = "source:cert-in:map";
}
internal sealed class CertInFetchJob : IJob
{
private readonly CertInConnector _connector;
public CertInFetchJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class CertInParseJob : IJob
{
private readonly CertInConnector _connector;
public CertInParseJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class CertInMapJob : IJob
{
private readonly CertInConnector _connector;
public CertInMapJob(CertInConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

58
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/PaginationPlanner.cs
View File

@@ -1,29 +1,29 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
/// </summary>
public static class PaginationPlanner
{
/// <summary>
/// Enumerates additional page start indices given the total result count returned by the source.
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
/// </summary>
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
{
if (totalResults <= 0 || resultsPerPage <= 0)
{
yield break;
}
if (firstPageStartIndex < 0)
{
firstPageStartIndex = 0;
}
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
{
yield return start;
}
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
/// </summary>
public static class PaginationPlanner
{
/// <summary>
/// Enumerates additional page start indices given the total result count returned by the source.
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
/// </summary>
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
{
if (totalResults <= 0 || resultsPerPage <= 0)
{
yield break;
}
if (firstPageStartIndex < 0)
{
firstPageStartIndex = 0;
}
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
{
yield return start;
}
}
}

86
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorOptions.cs
View File

@@ -1,43 +1,43 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Configuration applied when advancing sliding time-window cursors.
/// </summary>
public sealed class TimeWindowCursorOptions
{
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
public void EnsureValid()
{
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Window overlap cannot be negative.");
}
if (Overlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill must be positive.");
}
if (MinimumWindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Minimum window size must be positive.");
}
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Configuration applied when advancing sliding time-window cursors.
/// </summary>
public sealed class TimeWindowCursorOptions
{
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
public void EnsureValid()
{
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Window overlap cannot be negative.");
}
if (Overlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill must be positive.");
}
if (MinimumWindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Minimum window size must be positive.");
}
}
}

100
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorPlanner.cs
View File

@@ -1,50 +1,50 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Utility methods for computing sliding time-window ranges used by connectors.
/// </summary>
public static class TimeWindowCursorPlanner
{
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
{
ArgumentNullException.ThrowIfNull(options);
options.EnsureValid();
var effectiveState = state ?? TimeWindowCursorState.Empty;
var earliest = now - options.InitialBackfill;
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
if (anchorEnd < earliest)
{
anchorEnd = earliest;
}
var start = anchorEnd - options.Overlap;
if (start < earliest)
{
start = earliest;
}
var end = start + options.WindowSize;
if (end > now)
{
end = now;
}
if (end <= start)
{
end = start + options.MinimumWindowSize;
if (end > now)
{
end = now;
}
}
if (end <= start)
{
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
}
return new TimeWindow(start, end);
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Utility methods for computing sliding time-window ranges used by connectors.
/// </summary>
public static class TimeWindowCursorPlanner
{
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
{
ArgumentNullException.ThrowIfNull(options);
options.EnsureValid();
var effectiveState = state ?? TimeWindowCursorState.Empty;
var earliest = now - options.InitialBackfill;
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
if (anchorEnd < earliest)
{
anchorEnd = earliest;
}
var start = anchorEnd - options.Overlap;
if (start < earliest)
{
start = earliest;
}
var end = start + options.WindowSize;
if (end > now)
{
end = now;
}
if (end <= start)
{
end = start + options.MinimumWindowSize;
if (end > now)
{
end = now;
}
}
if (end <= start)
{
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
}
return new TimeWindow(start, end);
}
}

168
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs
View File

@@ -1,84 +1,84 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Represents the persisted state of a sliding time-window cursor.
/// </summary>
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
{
public static TimeWindowCursorState Empty { get; } = new(null, null);
public TimeWindowCursorState WithWindow(TimeWindow window)
{
return new TimeWindowCursorState(window.Start, window.End);
}
public BsonDocument ToBsonDocument(string startField = "windowStart", string endField = "windowEnd")
{
var document = new BsonDocument();
WriteTo(document, startField, endField);
return document;
}
public void WriteTo(BsonDocument document, string startField = "windowStart", string endField = "windowEnd")
{
ArgumentNullException.ThrowIfNull(document);
ArgumentException.ThrowIfNullOrEmpty(startField);
ArgumentException.ThrowIfNullOrEmpty(endField);
document.Remove(startField);
document.Remove(endField);
if (LastWindowStart.HasValue)
{
document[startField] = LastWindowStart.Value.UtcDateTime;
}
if (LastWindowEnd.HasValue)
{
document[endField] = LastWindowEnd.Value.UtcDateTime;
}
}
public static TimeWindowCursorState FromBsonDocument(BsonDocument? document, string startField = "windowStart", string endField = "windowEnd")
{
if (document is null)
{
return Empty;
}
DateTimeOffset? start = null;
DateTimeOffset? end = null;
if (document.TryGetValue(startField, out var startValue))
{
start = ReadDateTimeOffset(startValue);
}
if (document.TryGetValue(endField, out var endValue))
{
end = ReadDateTimeOffset(endValue);
}
return new TimeWindowCursorState(start, end);
}
private static DateTimeOffset? ReadDateTimeOffset(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
/// <summary>
/// Simple value object describing a time window.
/// </summary>
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
{
public TimeSpan Duration => End - Start;
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Represents the persisted state of a sliding time-window cursor.
/// </summary>
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
{
public static TimeWindowCursorState Empty { get; } = new(null, null);
public TimeWindowCursorState WithWindow(TimeWindow window)
{
return new TimeWindowCursorState(window.Start, window.End);
}
public DocumentObject ToDocumentObject(string startField = "windowStart", string endField = "windowEnd")
{
var document = new DocumentObject();
WriteTo(document, startField, endField);
return document;
}
public void WriteTo(DocumentObject document, string startField = "windowStart", string endField = "windowEnd")
{
ArgumentNullException.ThrowIfNull(document);
ArgumentException.ThrowIfNullOrEmpty(startField);
ArgumentException.ThrowIfNullOrEmpty(endField);
document.Remove(startField);
document.Remove(endField);
if (LastWindowStart.HasValue)
{
document[startField] = LastWindowStart.Value.UtcDateTime;
}
if (LastWindowEnd.HasValue)
{
document[endField] = LastWindowEnd.Value.UtcDateTime;
}
}
public static TimeWindowCursorState FromDocumentObject(DocumentObject? document, string startField = "windowStart", string endField = "windowEnd")
{
if (document is null)
{
return Empty;
}
DateTimeOffset? start = null;
DateTimeOffset? end = null;
if (document.TryGetValue(startField, out var startValue))
{
start = ReadDateTimeOffset(startValue);
}
if (document.TryGetValue(endField, out var endValue))
{
end = ReadDateTimeOffset(endValue);
}
return new TimeWindowCursorState(start, end);
}
private static DateTimeOffset? ReadDateTimeOffset(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
/// <summary>
/// Simple value object describing a time window.
/// </summary>
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
{
public TimeSpan Duration => End - Start;
}

54
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/DocumentStatuses.cs
View File

@@ -1,27 +1,27 @@
namespace StellaOps.Concelier.Connector.Common;
/// <summary>
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
/// </summary>
public static class DocumentStatuses
{
/// <summary>
/// Document captured from the upstream source and awaiting schema validation/parsing.
/// </summary>
public const string PendingParse = "pending-parse";
/// <summary>
/// Document parsed and sanitized; awaiting canonical mapping.
/// </summary>
public const string PendingMap = "pending-map";
/// <summary>
/// Document fully mapped to canonical advisories.
/// </summary>
public const string Mapped = "mapped";
/// <summary>
/// Document failed processing; requires manual intervention before retry.
/// </summary>
public const string Failed = "failed";
}
namespace StellaOps.Concelier.Connector.Common;
/// <summary>
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
/// </summary>
public static class DocumentStatuses
{
/// <summary>
/// Document captured from the upstream source and awaiting schema validation/parsing.
/// </summary>
public const string PendingParse = "pending-parse";
/// <summary>
/// Document parsed and sanitized; awaiting canonical mapping.
/// </summary>
public const string PendingMap = "pending-map";
/// <summary>
/// Document fully mapped to canonical advisories.
/// </summary>
public const string Mapped = "mapped";
/// <summary>
/// Document failed processing; requires manual intervention before retry.
/// </summary>
public const string Failed = "failed";
}

86
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/CryptoJitterSource.cs
View File

@@ -1,43 +1,43 @@
using System.Security.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
/// </summary>
public sealed class CryptoJitterSource : IJitterSource
{
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
{
if (maxInclusive < minInclusive)
{
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
}
if (minInclusive < TimeSpan.Zero)
{
minInclusive = TimeSpan.Zero;
}
if (maxInclusive == minInclusive)
{
return minInclusive;
}
var minTicks = minInclusive.Ticks;
var maxTicks = maxInclusive.Ticks;
var range = maxTicks - minTicks;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer);
var ratio = sample / (double)ulong.MaxValue;
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
if (jitterTicks > range)
{
jitterTicks = range;
}
return TimeSpan.FromTicks(minTicks + jitterTicks);
}
}
using System.Security.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
/// </summary>
public sealed class CryptoJitterSource : IJitterSource
{
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
{
if (maxInclusive < minInclusive)
{
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
}
if (minInclusive < TimeSpan.Zero)
{
minInclusive = TimeSpan.Zero;
}
if (maxInclusive == minInclusive)
{
return minInclusive;
}
var minTicks = minInclusive.Ticks;
var maxTicks = maxInclusive.Ticks;
var range = maxTicks - minTicks;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer);
var ratio = sample / (double)ulong.MaxValue;
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
if (jitterTicks > range)
{
jitterTicks = range;
}
return TimeSpan.FromTicks(minTicks + jitterTicks);
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/IJitterSource.cs
View File

@@ -1,9 +1,9 @@
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Produces random jitter durations used to decorrelate retries.
/// </summary>
public interface IJitterSource
{
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
}
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Produces random jitter durations used to decorrelate retries.
/// </summary>
public interface IJitterSource
{
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
}

46
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchContentResult.cs
View File

@@ -1,18 +1,18 @@
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Result of fetching raw response content without persisting a document.
/// </summary>
public sealed record SourceFetchContentResult
{
private SourceFetchContentResult(
HttpStatusCode statusCode,
byte[]? content,
bool notModified,
string? etag,
DateTimeOffset? lastModified,
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Result of fetching raw response content without persisting a document.
/// </summary>
public sealed record SourceFetchContentResult
{
private SourceFetchContentResult(
HttpStatusCode statusCode,
byte[]? content,
bool notModified,
string? etag,
DateTimeOffset? lastModified,
string? contentType,
int attempts,
IReadOnlyDictionary<string, string>? headers)
@@ -30,14 +30,14 @@ public sealed record SourceFetchContentResult
public HttpStatusCode StatusCode { get; }
public byte[]? Content { get; }
public bool IsSuccess => Content is not null;
public bool IsNotModified { get; }
public string? ETag { get; }
public DateTimeOffset? LastModified { get; }
public bool IsSuccess => Content is not null;
public bool IsNotModified { get; }
public string? ETag { get; }
public DateTimeOffset? LastModified { get; }
public string? ContentType { get; }

48
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchRequest.cs
View File

@@ -1,24 +1,24 @@
using System.Collections.Generic;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Parameters describing a fetch operation for a source connector.
/// </summary>
public sealed record SourceFetchRequest(
string ClientName,
string SourceName,
HttpMethod Method,
Uri RequestUri,
IReadOnlyDictionary<string, string>? Metadata = null,
string? ETag = null,
DateTimeOffset? LastModified = null,
TimeSpan? TimeoutOverride = null,
IReadOnlyList<string>? AcceptHeaders = null)
{
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
: this(clientName, sourceName, HttpMethod.Get, requestUri)
{
}
}
using System.Collections.Generic;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Parameters describing a fetch operation for a source connector.
/// </summary>
public sealed record SourceFetchRequest(
string ClientName,
string SourceName,
HttpMethod Method,
Uri RequestUri,
IReadOnlyDictionary<string, string>? Metadata = null,
string? ETag = null,
DateTimeOffset? LastModified = null,
TimeSpan? TimeoutOverride = null,
IReadOnlyList<string>? AcceptHeaders = null)
{
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
: this(clientName, sourceName, HttpMethod.Get, requestUri)
{
}
}

68
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchResult.cs
View File

@@ -1,34 +1,34 @@
using System.Net;
using StellaOps.Concelier.Storage.Contracts;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Outcome of fetching a raw document from an upstream source.
/// </summary>
public sealed record SourceFetchResult
{
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
{
StatusCode = statusCode;
Document = document;
IsNotModified = notModified;
}
public HttpStatusCode StatusCode { get; }
public StorageDocument? Document { get; }
public bool IsSuccess => Document is not null;
public bool IsNotModified { get; }
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
=> new(statusCode, document, notModified: false);
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: true);
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: false);
}
using System.Net;
using StellaOps.Concelier.Storage.Contracts;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Outcome of fetching a raw document from an upstream source.
/// </summary>
public sealed record SourceFetchResult
{
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
{
StatusCode = statusCode;
Document = document;
IsNotModified = notModified;
}
public HttpStatusCode StatusCode { get; }
public StorageDocument? Document { get; }
public bool IsSuccess => Document is not null;
public bool IsNotModified { get; }
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
=> new(statusCode, document, notModified: false);
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: true);
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: false);
}

1602
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs
View File

File diff suppressed because it is too large Load Diff

88
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceRetryPolicy.cs
View File

@@ -2,10 +2,10 @@ using System.Globalization;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Provides retry/backoff behavior for source HTTP fetches.
/// </summary>
/// <summary>
/// Provides retry/backoff behavior for source HTTP fetches.
/// </summary>
internal static class SourceRetryPolicy
{
private static readonly StringComparer HeaderComparer = StringComparer.OrdinalIgnoreCase;
@@ -15,34 +15,34 @@ internal static class SourceRetryPolicy
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> sender,
int maxAttempts,
TimeSpan baseDelay,
IJitterSource jitterSource,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
var attempt = 0;
while (true)
{
attempt++;
using var request = requestFactory();
HttpResponseMessage response;
try
{
response = await sender(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (attempt < maxAttempts)
{
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
IJitterSource jitterSource,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
var attempt = 0;
while (true)
{
attempt++;
using var request = requestFactory();
HttpResponseMessage response;
try
{
response = await sender(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (attempt < maxAttempts)
{
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
if (NeedsRetry(response) && attempt < maxAttempts)
{
var delay = ComputeDelay(
@@ -55,11 +55,11 @@ internal static class SourceRetryPolicy
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
return response;
}
}
return response;
}
}
private static bool NeedsRetry(HttpResponseMessage response)
{
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
@@ -76,13 +76,13 @@ internal static class SourceRetryPolicy
return status >= 500 && status < 600;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
return retryAfter.Value;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
return retryAfter.Value;
}
var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1));
var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250))
?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250));

308
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Html/HtmlContentSanitizer.cs
View File

@@ -1,79 +1,79 @@
using System.Linq;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Url;
namespace StellaOps.Concelier.Connector.Common.Html;
/// <summary>
/// Sanitizes untrusted HTML fragments produced by upstream advisories.
/// Removes executable content, enforces an allowlist of elements, and normalizes anchor href values.
/// </summary>
public sealed class HtmlContentSanitizer
{
private static readonly HashSet<string> AllowedElements = new(StringComparer.OrdinalIgnoreCase)
using System.Linq;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Url;
namespace StellaOps.Concelier.Connector.Common.Html;
/// <summary>
/// Sanitizes untrusted HTML fragments produced by upstream advisories.
/// Removes executable content, enforces an allowlist of elements, and normalizes anchor href values.
/// </summary>
public sealed class HtmlContentSanitizer
{
private static readonly HashSet<string> AllowedElements = new(StringComparer.OrdinalIgnoreCase)
{
"a", "abbr", "article", "b", "body", "blockquote", "br", "code", "dd", "div", "dl", "dt",
"em", "h1", "h2", "h3", "h4", "h5", "h6", "html", "i", "li", "ol", "p", "pre", "s",
"section", "small", "span", "strong", "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "ul"
};
private static readonly HashSet<string> UrlAttributes = new(StringComparer.OrdinalIgnoreCase)
{
"href", "src",
};
private readonly HtmlParser _parser;
public HtmlContentSanitizer()
{
_parser = new HtmlParser(new HtmlParserOptions
{
IsKeepingSourceReferences = false,
});
}
/// <summary>
/// Sanitizes <paramref name="html"/> and returns a safe fragment suitable for rendering.
/// </summary>
public string Sanitize(string? html, Uri? baseUri = null)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var document = _parser.ParseDocument(html);
if (document.Body is null)
{
return string.Empty;
}
foreach (var element in document.All.ToList())
{
if (IsDangerous(element))
{
element.Remove();
continue;
}
if (!AllowedElements.Contains(element.LocalName))
{
var owner = element.Owner;
if (owner is null)
{
element.Remove();
continue;
}
var text = element.TextContent ?? string.Empty;
element.Replace(owner.CreateTextNode(text));
continue;
}
CleanAttributes(element, baseUri);
}
private static readonly HashSet<string> UrlAttributes = new(StringComparer.OrdinalIgnoreCase)
{
"href", "src",
};
private readonly HtmlParser _parser;
public HtmlContentSanitizer()
{
_parser = new HtmlParser(new HtmlParserOptions
{
IsKeepingSourceReferences = false,
});
}
/// <summary>
/// Sanitizes <paramref name="html"/> and returns a safe fragment suitable for rendering.
/// </summary>
public string Sanitize(string? html, Uri? baseUri = null)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var document = _parser.ParseDocument(html);
if (document.Body is null)
{
return string.Empty;
}
foreach (var element in document.All.ToList())
{
if (IsDangerous(element))
{
element.Remove();
continue;
}
if (!AllowedElements.Contains(element.LocalName))
{
var owner = element.Owner;
if (owner is null)
{
element.Remove();
continue;
}
var text = element.TextContent ?? string.Empty;
element.Replace(owner.CreateTextNode(text));
continue;
}
CleanAttributes(element, baseUri);
}
var body = document.Body ?? document.DocumentElement;
if (body is null)
{
@@ -82,22 +82,22 @@ public sealed class HtmlContentSanitizer
var innerHtml = body.InnerHtml;
return string.IsNullOrWhiteSpace(innerHtml) ? string.Empty : innerHtml.Trim();
}
private static bool IsDangerous(IElement element)
{
if (string.Equals(element.LocalName, "script", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "style", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "iframe", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "object", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "embed", StringComparison.OrdinalIgnoreCase))
{
return true;
}
return false;
}
}
private static bool IsDangerous(IElement element)
{
if (string.Equals(element.LocalName, "script", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "style", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "iframe", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "object", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "embed", StringComparison.OrdinalIgnoreCase))
{
return true;
}
return false;
}
private static void CleanAttributes(IElement element, Uri? baseUri)
{
if (element.Attributes is null || element.Attributes.Length == 0)
@@ -111,70 +111,70 @@ public sealed class HtmlContentSanitizer
{
element.RemoveAttribute(attribute.Name);
continue;
}
if (UrlAttributes.Contains(attribute.Name))
{
NormalizeUrlAttribute(element, attribute, baseUri);
continue;
}
if (!IsAttributeAllowed(element.LocalName, attribute.Name))
{
element.RemoveAttribute(attribute.Name);
}
}
}
private static bool IsAttributeAllowed(string elementName, string attributeName)
{
if (string.Equals(attributeName, "title", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "a", StringComparison.OrdinalIgnoreCase)
&& string.Equals(attributeName, "rel", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "table", StringComparison.OrdinalIgnoreCase)
&& (string.Equals(attributeName, "border", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellpadding", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellspacing", StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return false;
}
private static void NormalizeUrlAttribute(IElement element, IAttr attribute, Uri? baseUri)
{
if (string.IsNullOrWhiteSpace(attribute.Value))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (!UrlNormalizer.TryNormalize(attribute.Value, baseUri, out var normalized))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (string.Equals(element.LocalName, "a", StringComparison.OrdinalIgnoreCase))
{
element.SetAttribute("rel", "noopener nofollow noreferrer");
}
if (normalized is null)
{
element.RemoveAttribute(attribute.Name);
return;
}
element.SetAttribute(attribute.Name, normalized.ToString());
}
}
}
if (UrlAttributes.Contains(attribute.Name))
{
NormalizeUrlAttribute(element, attribute, baseUri);
continue;
}
if (!IsAttributeAllowed(element.LocalName, attribute.Name))
{
element.RemoveAttribute(attribute.Name);
}
}
}
private static bool IsAttributeAllowed(string elementName, string attributeName)
{
if (string.Equals(attributeName, "title", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "a", StringComparison.OrdinalIgnoreCase)
&& string.Equals(attributeName, "rel", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "table", StringComparison.OrdinalIgnoreCase)
&& (string.Equals(attributeName, "border", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellpadding", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellspacing", StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return false;
}
private static void NormalizeUrlAttribute(IElement element, IAttr attribute, Uri? baseUri)
{
if (string.IsNullOrWhiteSpace(attribute.Value))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (!UrlNormalizer.TryNormalize(attribute.Value, baseUri, out var normalized))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (string.Equals(element.LocalName, "a", StringComparison.OrdinalIgnoreCase))
{
element.SetAttribute("rel", "noopener nofollow noreferrer");
}
if (normalized is null)
{
element.RemoveAttribute(attribute.Name);
return;
}
element.SetAttribute(attribute.Name, normalized.ToString());
}
}

72
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/AllowlistedHttpMessageHandler.cs
View File

@@ -1,36 +1,36 @@
using System.Net.Http.Headers;
namespace StellaOps.Concelier.Connector.Common.Http;
/// <summary>
/// Delegating handler that enforces an allowlist of destination hosts for outbound requests.
/// </summary>
internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler
{
private readonly IReadOnlyCollection<string> _allowedHosts;
public AllowlistedHttpMessageHandler(SourceHttpClientOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var snapshot = options.GetAllowedHostsSnapshot();
if (snapshot.Count == 0)
{
throw new InvalidOperationException("Source HTTP client must configure at least one allowed host.");
}
_allowedHosts = snapshot;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var host = request.RequestUri?.Host;
if (string.IsNullOrWhiteSpace(host) || !_allowedHosts.Contains(host))
{
throw new InvalidOperationException($"Request host '{host ?? "<null>"}' is not allowlisted for this source.");
}
return base.SendAsync(request, cancellationToken);
}
}
using System.Net.Http.Headers;
namespace StellaOps.Concelier.Connector.Common.Http;
/// <summary>
/// Delegating handler that enforces an allowlist of destination hosts for outbound requests.
/// </summary>
internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler
{
private readonly IReadOnlyCollection<string> _allowedHosts;
public AllowlistedHttpMessageHandler(SourceHttpClientOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var snapshot = options.GetAllowedHostsSnapshot();
if (snapshot.Count == 0)
{
throw new InvalidOperationException("Source HTTP client must configure at least one allowed host.");
}
_allowedHosts = snapshot;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var host = request.RequestUri?.Host;
if (string.IsNullOrWhiteSpace(host) || !_allowedHosts.Contains(host))
{
throw new InvalidOperationException($"Request host '{host ?? "<null>"}' is not allowlisted for this source.");
}
return base.SendAsync(request, cancellationToken);
}
}

652
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientConfigurationBinder.cs
View File

@@ -1,184 +1,184 @@
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.IO;
using System.Net.Security;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Http;
internal static class SourceHttpClientConfigurationBinder
{
private const string ConcelierSection = "concelier";
private const string HttpClientsSection = "httpClients";
private const string SourcesSection = "sources";
private const string HttpSection = "http";
private const string AllowInvalidKey = "allowInvalidCertificates";
private const string TrustedRootPathsKey = "trustedRootPaths";
private const string ProxySection = "proxy";
private const string ProxyAddressKey = "address";
private const string ProxyBypassOnLocalKey = "bypassOnLocal";
private const string ProxyBypassListKey = "bypassList";
private const string ProxyUseDefaultCredentialsKey = "useDefaultCredentials";
private const string ProxyUsernameKey = "username";
private const string ProxyPasswordKey = "password";
private const string OfflineRootKey = "offlineRoot";
private const string OfflineRootEnvironmentVariable = "CONCELIER_OFFLINE_ROOT";
public static void Apply(IServiceProvider services, string clientName, SourceHttpClientOptions options)
{
var configuration = services.GetService(typeof(IConfiguration)) as IConfiguration;
if (configuration is null)
{
return;
}
var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory;
var logger = loggerFactory?.CreateLogger("SourceHttpClientConfiguration");
var hostEnvironment = services.GetService(typeof(IHostEnvironment)) as IHostEnvironment;
var processed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var section in EnumerateCandidateSections(configuration, clientName))
{
if (section is null || !section.Exists() || !processed.Add(section.Path))
{
continue;
}
ApplySection(section, configuration, hostEnvironment, clientName, options, logger);
}
}
private static IEnumerable<IConfigurationSection> EnumerateCandidateSections(IConfiguration configuration, string clientName)
{
var names = BuildCandidateNames(clientName);
foreach (var name in names)
{
var httpClientSection = GetSection(configuration, ConcelierSection, HttpClientsSection, name);
if (httpClientSection is not null && httpClientSection.Exists())
{
yield return httpClientSection;
}
var sourceHttpSection = GetSection(configuration, ConcelierSection, SourcesSection, name, HttpSection);
if (sourceHttpSection is not null && sourceHttpSection.Exists())
{
yield return sourceHttpSection;
}
}
}
private static IEnumerable<string> BuildCandidateNames(string clientName)
{
yield return clientName;
if (clientName.StartsWith("source.", StringComparison.OrdinalIgnoreCase) && clientName.Length > "source.".Length)
{
yield return clientName["source.".Length..];
}
var noDots = clientName.Replace('.', '_');
if (!string.Equals(noDots, clientName, StringComparison.OrdinalIgnoreCase))
{
yield return noDots;
}
}
private static IConfigurationSection? GetSection(IConfiguration configuration, params string[] pathSegments)
{
IConfiguration? current = configuration;
foreach (var segment in pathSegments)
{
if (current is null)
{
return null;
}
current = current.GetSection(segment);
}
return current as IConfigurationSection;
}
private static void ApplySection(
IConfigurationSection section,
IConfiguration rootConfiguration,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var allowInvalid = section.GetValue<bool?>(AllowInvalidKey);
if (allowInvalid == true)
{
options.AllowInvalidServerCertificates = true;
var previous = options.ServerCertificateCustomValidation;
options.ServerCertificateCustomValidation = (certificate, chain, errors) =>
{
if (allowInvalid == true)
{
return true;
}
return previous?.Invoke(certificate, chain, errors) ?? errors == SslPolicyErrors.None;
};
logger?.LogWarning(
"Source HTTP client '{ClientName}' is configured to bypass TLS certificate validation.",
clientName);
}
var offlineRoot = section.GetValue<string?>(OfflineRootKey)
?? rootConfiguration.GetSection(ConcelierSection).GetValue<string?>(OfflineRootKey)
?? Environment.GetEnvironmentVariable(OfflineRootEnvironmentVariable);
ApplyTrustedRoots(section, offlineRoot, hostEnvironment, clientName, options, logger);
ApplyProxyConfiguration(section, clientName, options, logger);
}
private static void ApplyTrustedRoots(
IConfigurationSection section,
string? offlineRoot,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var trustedRootSection = section.GetSection(TrustedRootPathsKey);
if (!trustedRootSection.Exists())
{
return;
}
var paths = trustedRootSection.Get<string[]?>();
if (paths is null || paths.Length == 0)
{
return;
}
foreach (var rawPath in paths)
{
if (string.IsNullOrWhiteSpace(rawPath))
{
continue;
}
var resolvedPath = ResolvePath(rawPath, offlineRoot, hostEnvironment);
if (!File.Exists(resolvedPath))
{
var message = string.Format(
CultureInfo.InvariantCulture,
"Trusted root certificate '{0}' resolved to '{1}' but was not found.",
rawPath,
resolvedPath);
throw new FileNotFoundException(message, resolvedPath);
}
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.IO;
using System.Net.Security;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Http;
internal static class SourceHttpClientConfigurationBinder
{
private const string ConcelierSection = "concelier";
private const string HttpClientsSection = "httpClients";
private const string SourcesSection = "sources";
private const string HttpSection = "http";
private const string AllowInvalidKey = "allowInvalidCertificates";
private const string TrustedRootPathsKey = "trustedRootPaths";
private const string ProxySection = "proxy";
private const string ProxyAddressKey = "address";
private const string ProxyBypassOnLocalKey = "bypassOnLocal";
private const string ProxyBypassListKey = "bypassList";
private const string ProxyUseDefaultCredentialsKey = "useDefaultCredentials";
private const string ProxyUsernameKey = "username";
private const string ProxyPasswordKey = "password";
private const string OfflineRootKey = "offlineRoot";
private const string OfflineRootEnvironmentVariable = "CONCELIER_OFFLINE_ROOT";
public static void Apply(IServiceProvider services, string clientName, SourceHttpClientOptions options)
{
var configuration = services.GetService(typeof(IConfiguration)) as IConfiguration;
if (configuration is null)
{
return;
}
var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory;
var logger = loggerFactory?.CreateLogger("SourceHttpClientConfiguration");
var hostEnvironment = services.GetService(typeof(IHostEnvironment)) as IHostEnvironment;
var processed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var section in EnumerateCandidateSections(configuration, clientName))
{
if (section is null || !section.Exists() || !processed.Add(section.Path))
{
continue;
}
ApplySection(section, configuration, hostEnvironment, clientName, options, logger);
}
}
private static IEnumerable<IConfigurationSection> EnumerateCandidateSections(IConfiguration configuration, string clientName)
{
var names = BuildCandidateNames(clientName);
foreach (var name in names)
{
var httpClientSection = GetSection(configuration, ConcelierSection, HttpClientsSection, name);
if (httpClientSection is not null && httpClientSection.Exists())
{
yield return httpClientSection;
}
var sourceHttpSection = GetSection(configuration, ConcelierSection, SourcesSection, name, HttpSection);
if (sourceHttpSection is not null && sourceHttpSection.Exists())
{
yield return sourceHttpSection;
}
}
}
private static IEnumerable<string> BuildCandidateNames(string clientName)
{
yield return clientName;
if (clientName.StartsWith("source.", StringComparison.OrdinalIgnoreCase) && clientName.Length > "source.".Length)
{
yield return clientName["source.".Length..];
}
var noDots = clientName.Replace('.', '_');
if (!string.Equals(noDots, clientName, StringComparison.OrdinalIgnoreCase))
{
yield return noDots;
}
}
private static IConfigurationSection? GetSection(IConfiguration configuration, params string[] pathSegments)
{
IConfiguration? current = configuration;
foreach (var segment in pathSegments)
{
if (current is null)
{
return null;
}
current = current.GetSection(segment);
}
return current as IConfigurationSection;
}
private static void ApplySection(
IConfigurationSection section,
IConfiguration rootConfiguration,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var allowInvalid = section.GetValue<bool?>(AllowInvalidKey);
if (allowInvalid == true)
{
options.AllowInvalidServerCertificates = true;
var previous = options.ServerCertificateCustomValidation;
options.ServerCertificateCustomValidation = (certificate, chain, errors) =>
{
if (allowInvalid == true)
{
return true;
}
return previous?.Invoke(certificate, chain, errors) ?? errors == SslPolicyErrors.None;
};
logger?.LogWarning(
"Source HTTP client '{ClientName}' is configured to bypass TLS certificate validation.",
clientName);
}
var offlineRoot = section.GetValue<string?>(OfflineRootKey)
?? rootConfiguration.GetSection(ConcelierSection).GetValue<string?>(OfflineRootKey)
?? Environment.GetEnvironmentVariable(OfflineRootEnvironmentVariable);
ApplyTrustedRoots(section, offlineRoot, hostEnvironment, clientName, options, logger);
ApplyProxyConfiguration(section, clientName, options, logger);
}
private static void ApplyTrustedRoots(
IConfigurationSection section,
string? offlineRoot,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var trustedRootSection = section.GetSection(TrustedRootPathsKey);
if (!trustedRootSection.Exists())
{
return;
}
var paths = trustedRootSection.Get<string[]?>();
if (paths is null || paths.Length == 0)
{
return;
}
foreach (var rawPath in paths)
{
if (string.IsNullOrWhiteSpace(rawPath))
{
continue;
}
var resolvedPath = ResolvePath(rawPath, offlineRoot, hostEnvironment);
if (!File.Exists(resolvedPath))
{
var message = string.Format(
CultureInfo.InvariantCulture,
"Trusted root certificate '{0}' resolved to '{1}' but was not found.",
rawPath,
resolvedPath);
throw new FileNotFoundException(message, resolvedPath);
}
foreach (var certificate in LoadCertificates(resolvedPath))
{
var thumbprint = certificate.Thumbprint;
@@ -194,134 +194,134 @@ internal static class SourceHttpClientConfigurationBinder
}
}
}
private static void ApplyProxyConfiguration(
IConfigurationSection section,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var proxySection = section.GetSection(ProxySection);
if (!proxySection.Exists())
{
return;
}
var address = proxySection.GetValue<string?>(ProxyAddressKey);
if (!string.IsNullOrWhiteSpace(address))
{
if (Uri.TryCreate(address, UriKind.Absolute, out var uri))
{
options.ProxyAddress = uri;
}
else
{
logger?.LogWarning(
"Source HTTP client '{ClientName}' has invalid proxy address '{ProxyAddress}'.",
clientName,
address);
}
}
var bypassOnLocal = proxySection.GetValue<bool?>(ProxyBypassOnLocalKey);
if (bypassOnLocal.HasValue)
{
options.ProxyBypassOnLocal = bypassOnLocal.Value;
}
var bypassListSection = proxySection.GetSection(ProxyBypassListKey);
if (bypassListSection.Exists())
{
var entries = bypassListSection.Get<string[]?>();
options.ProxyBypassList.Clear();
if (entries is not null)
{
foreach (var entry in entries)
{
if (!string.IsNullOrWhiteSpace(entry))
{
options.ProxyBypassList.Add(entry.Trim());
}
}
}
}
var useDefaultCredentials = proxySection.GetValue<bool?>(ProxyUseDefaultCredentialsKey);
if (useDefaultCredentials.HasValue)
{
options.ProxyUseDefaultCredentials = useDefaultCredentials.Value;
}
var username = proxySection.GetValue<string?>(ProxyUsernameKey);
if (!string.IsNullOrWhiteSpace(username))
{
options.ProxyUsername = username.Trim();
}
var password = proxySection.GetValue<string?>(ProxyPasswordKey);
if (!string.IsNullOrWhiteSpace(password))
{
options.ProxyPassword = password;
}
}
private static string ResolvePath(string path, string? offlineRoot, IHostEnvironment? hostEnvironment)
{
if (Path.IsPathRooted(path))
{
return path;
}
if (!string.IsNullOrWhiteSpace(offlineRoot))
{
return Path.GetFullPath(Path.Combine(offlineRoot!, path));
}
var baseDirectory = hostEnvironment?.ContentRootPath ?? AppContext.BaseDirectory;
return Path.GetFullPath(Path.Combine(baseDirectory, path));
}
private static IEnumerable<X509Certificate2> LoadCertificates(string path)
{
var certificates = new List<X509Certificate2>();
var extension = Path.GetExtension(path);
if (extension.Equals(".pem", StringComparison.OrdinalIgnoreCase) || extension.Equals(".crt", StringComparison.OrdinalIgnoreCase))
{
var collection = new X509Certificate2Collection();
try
{
collection.ImportFromPemFile(path);
}
catch (CryptographicException)
{
collection.Clear();
}
if (collection.Count > 0)
{
foreach (var certificate in collection)
{
certificates.Add(certificate.CopyWithPrivateKeyIfAvailable());
}
}
else
{
certificates.Add(X509Certificate2.CreateFromPemFile(path));
}
}
else
{
// Use X509CertificateLoader to load certificates from PKCS#12 files (.pfx, .p12, etc.)
var certificate = System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12(
File.ReadAllBytes(path),
password: null);
certificates.Add(certificate);
}
return certificates;
}
private static void ApplyProxyConfiguration(
IConfigurationSection section,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var proxySection = section.GetSection(ProxySection);
if (!proxySection.Exists())
{
return;
}
var address = proxySection.GetValue<string?>(ProxyAddressKey);
if (!string.IsNullOrWhiteSpace(address))
{
if (Uri.TryCreate(address, UriKind.Absolute, out var uri))
{
options.ProxyAddress = uri;
}
else
{
logger?.LogWarning(
"Source HTTP client '{ClientName}' has invalid proxy address '{ProxyAddress}'.",
clientName,
address);
}
}
var bypassOnLocal = proxySection.GetValue<bool?>(ProxyBypassOnLocalKey);
if (bypassOnLocal.HasValue)
{
options.ProxyBypassOnLocal = bypassOnLocal.Value;
}
var bypassListSection = proxySection.GetSection(ProxyBypassListKey);
if (bypassListSection.Exists())
{
var entries = bypassListSection.Get<string[]?>();
options.ProxyBypassList.Clear();
if (entries is not null)
{
foreach (var entry in entries)
{
if (!string.IsNullOrWhiteSpace(entry))
{
options.ProxyBypassList.Add(entry.Trim());
}
}
}
}
var useDefaultCredentials = proxySection.GetValue<bool?>(ProxyUseDefaultCredentialsKey);
if (useDefaultCredentials.HasValue)
{
options.ProxyUseDefaultCredentials = useDefaultCredentials.Value;
}
var username = proxySection.GetValue<string?>(ProxyUsernameKey);
if (!string.IsNullOrWhiteSpace(username))
{
options.ProxyUsername = username.Trim();
}
var password = proxySection.GetValue<string?>(ProxyPasswordKey);
if (!string.IsNullOrWhiteSpace(password))
{
options.ProxyPassword = password;
}
}
private static string ResolvePath(string path, string? offlineRoot, IHostEnvironment? hostEnvironment)
{
if (Path.IsPathRooted(path))
{
return path;
}
if (!string.IsNullOrWhiteSpace(offlineRoot))
{
return Path.GetFullPath(Path.Combine(offlineRoot!, path));
}
var baseDirectory = hostEnvironment?.ContentRootPath ?? AppContext.BaseDirectory;
return Path.GetFullPath(Path.Combine(baseDirectory, path));
}
private static IEnumerable<X509Certificate2> LoadCertificates(string path)
{
var certificates = new List<X509Certificate2>();
var extension = Path.GetExtension(path);
if (extension.Equals(".pem", StringComparison.OrdinalIgnoreCase) || extension.Equals(".crt", StringComparison.OrdinalIgnoreCase))
{
var collection = new X509Certificate2Collection();
try
{
collection.ImportFromPemFile(path);
}
catch (CryptographicException)
{
collection.Clear();
}
if (collection.Count > 0)
{
foreach (var certificate in collection)
{
certificates.Add(certificate.CopyWithPrivateKeyIfAvailable());
}
}
else
{
certificates.Add(X509Certificate2.CreateFromPemFile(path));
}
}
else
{
// Use X509CertificateLoader to load certificates from PKCS#12 files (.pfx, .p12, etc.)
var certificate = System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12(
File.ReadAllBytes(path),
password: null);
certificates.Add(certificate);
}
return certificates;
}
private static bool AddTrustedCertificate(SourceHttpClientOptions options, X509Certificate2 certificate)
{
if (certificate is null)
@@ -347,20 +347,20 @@ internal static class SourceHttpClientConfigurationBinder
return true;
}
// Helper extension method to copy certificate (preserves private key if present)
private static X509Certificate2 CopyWithPrivateKeyIfAvailable(this X509Certificate2 certificate)
{
// In .NET 9+, use X509CertificateLoader instead of obsolete constructors
if (certificate.HasPrivateKey)
{
// Export with private key and re-import using X509CertificateLoader
var exported = certificate.Export(X509ContentType.Pkcs12);
return X509CertificateLoader.LoadPkcs12(exported, password: null);
}
else
{
// For certificates without private keys, load from raw data
return X509CertificateLoader.LoadCertificate(certificate.RawData);
}
}
}
// Helper extension method to copy certificate (preserves private key if present)
private static X509Certificate2 CopyWithPrivateKeyIfAvailable(this X509Certificate2 certificate)
{
// In .NET 9+, use X509CertificateLoader instead of obsolete constructors
if (certificate.HasPrivateKey)
{
// Export with private key and re-import using X509CertificateLoader
var exported = certificate.Export(X509ContentType.Pkcs12);
return X509CertificateLoader.LoadPkcs12(exported, password: null);
}
else
{
// For certificates without private keys, load from raw data
return X509CertificateLoader.LoadCertificate(certificate.RawData);
}
}
}

78
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientOptions.cs
View File

@@ -8,43 +8,43 @@ namespace StellaOps.Concelier.Connector.Common.Http;
/// <summary>
/// Configuration applied to named HTTP clients used by connectors.
/// </summary>
public sealed class SourceHttpClientOptions
{
private readonly HashSet<string> _allowedHosts = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, string> _defaultHeaders = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Gets or sets the base address used for relative requests.
/// </summary>
public Uri? BaseAddress { get; set; }
/// <summary>
/// Gets or sets the client timeout.
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Gets or sets the user-agent string applied to outgoing requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps.Concelier/1.0";
/// <summary>
/// Gets or sets whether redirects are allowed. Defaults to <c>true</c>.
/// </summary>
public bool AllowAutoRedirect { get; set; } = true;
/// <summary>
/// </summary>
public sealed class SourceHttpClientOptions
{
private readonly HashSet<string> _allowedHosts = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, string> _defaultHeaders = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Gets or sets the base address used for relative requests.
/// </summary>
public Uri? BaseAddress { get; set; }
/// <summary>
/// Gets or sets the client timeout.
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Gets or sets the user-agent string applied to outgoing requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps.Concelier/1.0";
/// <summary>
/// Gets or sets whether redirects are allowed. Defaults to <c>true</c>.
/// </summary>
public bool AllowAutoRedirect { get; set; } = true;
/// <summary>
/// Maximum number of retry attempts for transient failures.
/// </summary>
public int MaxAttempts { get; set; } = 3;
/// <summary>
/// Base delay applied to the exponential backoff policy.
/// </summary>
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Base delay applied to the exponential backoff policy.
/// </summary>
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Hosts that this client is allowed to contact.
/// </summary>
public ISet<string> AllowedHosts => _allowedHosts;
@@ -120,10 +120,10 @@ public sealed class SourceHttpClientOptions
public IDictionary<string, string> DefaultRequestHeaders => _defaultHeaders;
internal SourceHttpClientOptions Clone()
{
var clone = new SourceHttpClientOptions
{
BaseAddress = BaseAddress,
{
var clone = new SourceHttpClientOptions
{
BaseAddress = BaseAddress,
Timeout = Timeout,
UserAgent = UserAgent,
AllowAutoRedirect = AllowAutoRedirect,
@@ -145,8 +145,8 @@ public sealed class SourceHttpClientOptions
foreach (var host in _allowedHosts)
{
clone.AllowedHosts.Add(host);
}
}
foreach (var header in _defaultHeaders)
{
clone.DefaultRequestHeaders[header.Key] = header.Value;
@@ -167,4 +167,4 @@ public sealed class SourceHttpClientOptions
internal IReadOnlyCollection<string> GetAllowedHostsSnapshot()
=> new ReadOnlyCollection<string>(_allowedHosts.ToArray());
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/IJsonSchemaValidator.cs
View File

@@ -1,9 +1,9 @@
using System.Text.Json;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Common.Json;
public interface IJsonSchemaValidator
{
void Validate(JsonDocument document, JsonSchema schema, string documentName);
}
using System.Text.Json;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Common.Json;
public interface IJsonSchemaValidator
{
void Validate(JsonDocument document, JsonSchema schema, string documentName);
}

14
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationError.cs
View File

@@ -1,7 +1,7 @@
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed record JsonSchemaValidationError(
string InstanceLocation,
string SchemaLocation,
string Message,
string Keyword);
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed record JsonSchemaValidationError(
string InstanceLocation,
string SchemaLocation,
string Message,
string Keyword);

30
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationException.cs
View File

@@ -1,15 +1,15 @@
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidationException : Exception
{
public JsonSchemaValidationException(string documentName, IReadOnlyList<JsonSchemaValidationError> errors)
: base($"JSON schema validation failed for '{documentName}'.")
{
DocumentName = documentName;
Errors = errors ?? Array.Empty<JsonSchemaValidationError>();
}
public string DocumentName { get; }
public IReadOnlyList<JsonSchemaValidationError> Errors { get; }
}
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidationException : Exception
{
public JsonSchemaValidationException(string documentName, IReadOnlyList<JsonSchemaValidationError> errors)
: base($"JSON schema validation failed for '{documentName}'.")
{
DocumentName = documentName;
Errors = errors ?? Array.Empty<JsonSchemaValidationError>();
}
public string DocumentName { get; }
public IReadOnlyList<JsonSchemaValidationError> Errors { get; }
}

184
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidator.cs
View File

@@ -1,92 +1,92 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Json.Schema;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidator : IJsonSchemaValidator
{
private readonly ILogger<JsonSchemaValidator> _logger;
private const int MaxLoggedErrors = 5;
public JsonSchemaValidator(ILogger<JsonSchemaValidator> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public void Validate(JsonDocument document, JsonSchema schema, string documentName)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(schema);
ArgumentException.ThrowIfNullOrEmpty(documentName);
var result = schema.Evaluate(document.RootElement, new EvaluationOptions
{
OutputFormat = OutputFormat.List,
RequireFormatValidation = true,
});
if (result.IsValid)
{
return;
}
var errors = CollectErrors(result);
if (errors.Count == 0)
{
_logger.LogWarning("Schema validation failed for {Document} with unknown errors", documentName);
throw new JsonSchemaValidationException(documentName, errors);
}
foreach (var violation in errors.Take(MaxLoggedErrors))
{
_logger.LogWarning(
"Schema violation for {Document} at {InstanceLocation} (keyword: {Keyword}): {Message}",
documentName,
string.IsNullOrEmpty(violation.InstanceLocation) ? "#" : violation.InstanceLocation,
violation.Keyword,
violation.Message);
}
if (errors.Count > MaxLoggedErrors)
{
_logger.LogWarning("{Count} additional schema violations for {Document} suppressed", errors.Count - MaxLoggedErrors, documentName);
}
throw new JsonSchemaValidationException(documentName, errors);
}
private static IReadOnlyList<JsonSchemaValidationError> CollectErrors(EvaluationResults result)
{
var errors = new List<JsonSchemaValidationError>();
Aggregate(result, errors);
return errors;
}
private static void Aggregate(EvaluationResults node, List<JsonSchemaValidationError> errors)
{
if (node.Errors is { Count: > 0 })
{
foreach (var kvp in node.Errors)
{
errors.Add(new JsonSchemaValidationError(
node.InstanceLocation?.ToString() ?? string.Empty,
node.SchemaLocation?.ToString() ?? string.Empty,
kvp.Value,
kvp.Key));
}
}
if (node.Details is null)
{
return;
}
foreach (var child in node.Details)
{
Aggregate(child, errors);
}
}
}
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Json.Schema;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidator : IJsonSchemaValidator
{
private readonly ILogger<JsonSchemaValidator> _logger;
private const int MaxLoggedErrors = 5;
public JsonSchemaValidator(ILogger<JsonSchemaValidator> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public void Validate(JsonDocument document, JsonSchema schema, string documentName)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(schema);
ArgumentException.ThrowIfNullOrEmpty(documentName);
var result = schema.Evaluate(document.RootElement, new EvaluationOptions
{
OutputFormat = OutputFormat.List,
RequireFormatValidation = true,
});
if (result.IsValid)
{
return;
}
var errors = CollectErrors(result);
if (errors.Count == 0)
{
_logger.LogWarning("Schema validation failed for {Document} with unknown errors", documentName);
throw new JsonSchemaValidationException(documentName, errors);
}
foreach (var violation in errors.Take(MaxLoggedErrors))
{
_logger.LogWarning(
"Schema violation for {Document} at {InstanceLocation} (keyword: {Keyword}): {Message}",
documentName,
string.IsNullOrEmpty(violation.InstanceLocation) ? "#" : violation.InstanceLocation,
violation.Keyword,
violation.Message);
}
if (errors.Count > MaxLoggedErrors)
{
_logger.LogWarning("{Count} additional schema violations for {Document} suppressed", errors.Count - MaxLoggedErrors, documentName);
}
throw new JsonSchemaValidationException(documentName, errors);
}
private static IReadOnlyList<JsonSchemaValidationError> CollectErrors(EvaluationResults result)
{
var errors = new List<JsonSchemaValidationError>();
Aggregate(result, errors);
return errors;
}
private static void Aggregate(EvaluationResults node, List<JsonSchemaValidationError> errors)
{
if (node.Errors is { Count: > 0 })
{
foreach (var kvp in node.Errors)
{
errors.Add(new JsonSchemaValidationError(
node.InstanceLocation?.ToString() ?? string.Empty,
node.SchemaLocation?.ToString() ?? string.Empty,
kvp.Value,
kvp.Key));
}
}
if (node.Details is null)
{
return;
}
foreach (var child in node.Details)
{
Aggregate(child, errors);
}
}
}

242
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Packages/PackageCoordinateHelper.cs
View File

@@ -1,23 +1,23 @@
using System.Linq;
using System.Text;
using NuGet.Versioning;
using StellaOps.Concelier.Normalization.Identifiers;
namespace StellaOps.Concelier.Connector.Common.Packages;
/// <summary>
/// Shared helpers for working with Package URLs and SemVer coordinates inside connectors.
/// </summary>
public static class PackageCoordinateHelper
{
public static bool TryParsePackageUrl(string? value, out PackageCoordinates? coordinates)
{
coordinates = null;
if (!IdentifierNormalizer.TryNormalizePackageUrl(value, out var canonical, out var packageUrl) || packageUrl is null)
{
return false;
}
using System.Linq;
using System.Text;
using NuGet.Versioning;
using StellaOps.Concelier.Normalization.Identifiers;
namespace StellaOps.Concelier.Connector.Common.Packages;
/// <summary>
/// Shared helpers for working with Package URLs and SemVer coordinates inside connectors.
/// </summary>
public static class PackageCoordinateHelper
{
public static bool TryParsePackageUrl(string? value, out PackageCoordinates? coordinates)
{
coordinates = null;
if (!IdentifierNormalizer.TryNormalizePackageUrl(value, out var canonical, out var packageUrl) || packageUrl is null)
{
return false;
}
var namespaceSegments = packageUrl.NamespaceSegments.ToArray();
var subpathSegments = packageUrl.SubpathSegments.ToArray();
var qualifiers = packageUrl.Qualifiers.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase);
@@ -39,46 +39,46 @@ public static class PackageCoordinateHelper
SubpathSegments: subpathSegments,
Original: packageUrl.Original);
return true;
}
public static PackageCoordinates ParsePackageUrl(string value)
{
if (!TryParsePackageUrl(value, out var coordinates) || coordinates is null)
{
throw new FormatException($"Value '{value}' is not a valid Package URL");
}
return coordinates;
}
public static bool TryParseSemVer(string? value, out SemanticVersion? version, out string? normalized)
{
version = null;
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!SemanticVersion.TryParse(value.Trim(), out var parsed))
{
return false;
}
version = parsed;
normalized = parsed.ToNormalizedString();
return true;
}
public static bool TryParseSemVerRange(string? value, out VersionRange? range)
{
range = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
}
public static PackageCoordinates ParsePackageUrl(string value)
{
if (!TryParsePackageUrl(value, out var coordinates) || coordinates is null)
{
throw new FormatException($"Value '{value}' is not a valid Package URL");
}
return coordinates;
}
public static bool TryParseSemVer(string? value, out SemanticVersion? version, out string? normalized)
{
version = null;
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!SemanticVersion.TryParse(value.Trim(), out var parsed))
{
return false;
}
version = parsed;
normalized = parsed.ToNormalizedString();
return true;
}
public static bool TryParseSemVerRange(string? value, out VersionRange? range)
{
range = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
var trimmed = value.Trim();
if (trimmed.StartsWith("^", StringComparison.Ordinal))
{
@@ -113,57 +113,57 @@ public static class PackageCoordinateHelper
range = parsed;
return true;
}
public static string BuildPackageUrl(
string type,
IReadOnlyList<string>? namespaceSegments,
string name,
string? version = null,
IReadOnlyDictionary<string, string>? qualifiers = null,
IReadOnlyList<string>? subpathSegments = null)
{
ArgumentException.ThrowIfNullOrEmpty(type);
ArgumentException.ThrowIfNullOrEmpty(name);
var builder = new StringBuilder("pkg:");
builder.Append(type.Trim().ToLowerInvariant());
builder.Append('/');
if (namespaceSegments is not null && namespaceSegments.Count > 0)
{
builder.Append(string.Join('/', namespaceSegments.Select(NormalizeSegment)));
builder.Append('/');
}
builder.Append(NormalizeSegment(name));
if (!string.IsNullOrWhiteSpace(version))
{
builder.Append('@');
builder.Append(version.Trim());
}
if (qualifiers is not null && qualifiers.Count > 0)
{
builder.Append('?');
builder.Append(string.Join('&', qualifiers
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
.Select(kvp => $"{NormalizeSegment(kvp.Key)}={NormalizeSegment(kvp.Value)}")));
}
if (subpathSegments is not null && subpathSegments.Count > 0)
{
builder.Append('#');
builder.Append(string.Join('/', subpathSegments.Select(NormalizeSegment)));
}
return builder.ToString();
}
private static string NormalizeSegment(string value)
{
ArgumentNullException.ThrowIfNull(value);
}
public static string BuildPackageUrl(
string type,
IReadOnlyList<string>? namespaceSegments,
string name,
string? version = null,
IReadOnlyDictionary<string, string>? qualifiers = null,
IReadOnlyList<string>? subpathSegments = null)
{
ArgumentException.ThrowIfNullOrEmpty(type);
ArgumentException.ThrowIfNullOrEmpty(name);
var builder = new StringBuilder("pkg:");
builder.Append(type.Trim().ToLowerInvariant());
builder.Append('/');
if (namespaceSegments is not null && namespaceSegments.Count > 0)
{
builder.Append(string.Join('/', namespaceSegments.Select(NormalizeSegment)));
builder.Append('/');
}
builder.Append(NormalizeSegment(name));
if (!string.IsNullOrWhiteSpace(version))
{
builder.Append('@');
builder.Append(version.Trim());
}
if (qualifiers is not null && qualifiers.Count > 0)
{
builder.Append('?');
builder.Append(string.Join('&', qualifiers
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
.Select(kvp => $"{NormalizeSegment(kvp.Key)}={NormalizeSegment(kvp.Value)}")));
}
if (subpathSegments is not null && subpathSegments.Count > 0)
{
builder.Append('#');
builder.Append(string.Join('/', subpathSegments.Select(NormalizeSegment)));
}
return builder.ToString();
}
private static string NormalizeSegment(string value)
{
ArgumentNullException.ThrowIfNull(value);
var trimmed = value.Trim();
var unescaped = Uri.UnescapeDataString(trimmed);
var encoded = Uri.EscapeDataString(unescaped);
@@ -185,13 +185,13 @@ public static class PackageCoordinateHelper
return new SemanticVersion(0, 0, baseVersion.Patch + 1);
}
}
public sealed record PackageCoordinates(
string Canonical,
string Type,
IReadOnlyList<string> NamespaceSegments,
string Name,
string? Version,
IReadOnlyDictionary<string, string> Qualifiers,
IReadOnlyList<string> SubpathSegments,
string Original);
public sealed record PackageCoordinates(
string Canonical,
string Type,
IReadOnlyList<string> NamespaceSegments,
string Name,
string? Version,
IReadOnlyDictionary<string, string> Qualifiers,
IReadOnlyList<string> SubpathSegments,
string Original);

144
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Pdf/PdfTextExtractor.cs
View File

@@ -2,22 +2,22 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using System.Text;
using UglyToad.PdfPig;
using UglyToad.PdfPig.Content;
namespace StellaOps.Concelier.Connector.Common.Pdf;
/// <summary>
/// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies.
/// </summary>
public sealed class PdfTextExtractor
{
public async Task<PdfExtractionResult> ExtractTextAsync(Stream pdfStream, PdfExtractionOptions? options = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(pdfStream);
options ??= PdfExtractionOptions.Default;
using System.Text;
using UglyToad.PdfPig;
using UglyToad.PdfPig.Content;
namespace StellaOps.Concelier.Connector.Common.Pdf;
/// <summary>
/// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies.
/// </summary>
public sealed class PdfTextExtractor
{
public async Task<PdfExtractionResult> ExtractTextAsync(Stream pdfStream, PdfExtractionOptions? options = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(pdfStream);
options ??= PdfExtractionOptions.Default;
using var buffer = new MemoryStream();
await pdfStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
var rawBytes = buffer.ToArray();
@@ -28,10 +28,10 @@ public sealed class PdfTextExtractor
ClipPaths = true,
UseLenientParsing = true,
});
var builder = new StringBuilder();
var pageCount = 0;
var builder = new StringBuilder();
var pageCount = 0;
var totalPages = document.NumberOfPages;
for (var index = 1; index <= totalPages; index++)
{
@@ -52,12 +52,12 @@ public sealed class PdfTextExtractor
{
break;
}
if (pageCount > 1 && options.PageSeparator is not null)
{
builder.Append(options.PageSeparator);
}
if (pageCount > 1 && options.PageSeparator is not null)
{
builder.Append(options.PageSeparator);
}
string text;
try
{
@@ -93,8 +93,8 @@ public sealed class PdfTextExtractor
{
builder.AppendLine(text.Trim());
}
}
}
if (builder.Length == 0)
{
var raw = Encoding.ASCII.GetString(rawBytes);
@@ -119,28 +119,28 @@ public sealed class PdfTextExtractor
}
return new PdfExtractionResult(builder.ToString().Trim(), pageCount);
}
private static string FlattenWords(IEnumerable<Word> words)
{
var builder = new StringBuilder();
var first = true;
foreach (var word in words)
{
if (string.IsNullOrWhiteSpace(word.Text))
{
continue;
}
if (!first)
{
builder.Append(' ');
}
builder.Append(word.Text.Trim());
first = false;
}
}
private static string FlattenWords(IEnumerable<Word> words)
{
var builder = new StringBuilder();
var first = true;
foreach (var word in words)
{
if (string.IsNullOrWhiteSpace(word.Text))
{
continue;
}
if (!first)
{
builder.Append(' ');
}
builder.Append(word.Text.Trim());
first = false;
}
return builder.ToString();
}
@@ -160,25 +160,25 @@ public sealed class PdfTextExtractor
return builder.ToString();
}
}
public sealed record PdfExtractionResult(string Text, int PagesProcessed);
public sealed record PdfExtractionOptions
{
public static PdfExtractionOptions Default { get; } = new();
/// <summary>
/// Maximum number of pages to read. Null reads the entire document.
/// </summary>
public int? MaxPages { get; init; }
/// <summary>
/// When true, uses PdfPig's native layout text. When false, collapses to a single line per page.
/// </summary>
public bool PreserveLayout { get; init; } = true;
/// <summary>
/// Separator inserted between pages. Null disables separators.
/// </summary>
public string? PageSeparator { get; init; } = "\n\n";
}
public sealed record PdfExtractionResult(string Text, int PagesProcessed);
public sealed record PdfExtractionOptions
{
public static PdfExtractionOptions Default { get; } = new();
/// <summary>
/// Maximum number of pages to read. Null reads the entire document.
/// </summary>
public int? MaxPages { get; init; }
/// <summary>
/// When true, uses PdfPig's native layout text. When false, collapses to a single line per page.
/// </summary>
public bool PreserveLayout { get; init; } = true;
/// <summary>
/// Separator inserted between pages. Null disables separators.
/// </summary>
public string? PageSeparator { get; init; } = "\n\n";
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Common.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Common.Tests")]

318
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedModels.cs
View File

@@ -1,159 +1,159 @@
using StellaOps.Concelier.Connector.Common;
namespace StellaOps.Concelier.Connector.Common.State;
/// <summary>
/// Describes a raw upstream document that should be persisted for a connector during seeding.
/// </summary>
public sealed record SourceStateSeedDocument
{
/// <summary>
/// Absolute source URI. Must match the connector's upstream document identifier.
/// </summary>
public string Uri { get; init; } = string.Empty;
/// <summary>
/// Raw document payload. Required when creating or replacing a document.
/// </summary>
public byte[] Content { get; init; } = Array.Empty<byte>();
/// <summary>
/// Optional explicit document identifier. When provided it overrides auto-generated IDs.
/// </summary>
public Guid? DocumentId { get; init; }
/// <summary>
/// MIME type for the document payload.
/// </summary>
public string? ContentType { get; init; }
/// <summary>
/// Status assigned to the document. Defaults to <see cref="DocumentStatuses.PendingParse"/>.
/// </summary>
public string Status { get; init; } = DocumentStatuses.PendingParse;
/// <summary>
/// Optional HTTP-style headers persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Headers { get; init; }
/// <summary>
/// Source metadata (connector specific) persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
/// <summary>
/// Upstream ETag value, if available.
/// </summary>
public string? Etag { get; init; }
/// <summary>
/// Upstream last-modified timestamp, if available.
/// </summary>
public DateTimeOffset? LastModified { get; init; }
/// <summary>
/// Optional document expiration. When set a TTL will purge the raw payload after the configured retention.
/// </summary>
public DateTimeOffset? ExpiresAt { get; init; }
/// <summary>
/// Fetch timestamp stamped onto the document. Defaults to the seed completion timestamp.
/// </summary>
public DateTimeOffset? FetchedAt { get; init; }
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingDocuments</c> set.
/// </summary>
public bool AddToPendingDocuments { get; init; } = true;
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingMappings</c> set.
/// </summary>
public bool AddToPendingMappings { get; init; }
/// <summary>
/// Optional identifiers that should be recorded on the cursor to avoid duplicate ingestion.
/// </summary>
public IReadOnlyCollection<string>? KnownIdentifiers { get; init; }
}
/// <summary>
/// Cursor updates that should accompany seeded documents.
/// </summary>
public sealed record SourceStateSeedCursor
{
/// <summary>
/// Optional <c>pendingDocuments</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingDocuments { get; init; }
/// <summary>
/// Optional <c>pendingMappings</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingMappings { get; init; }
/// <summary>
/// Optional known advisory identifiers to merge with the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Upstream window watermark tracked by connectors that rely on last-modified cursors.
/// </summary>
public DateTimeOffset? LastModifiedCursor { get; init; }
/// <summary>
/// Optional fetch timestamp used by connectors that track the last polling instant.
/// </summary>
public DateTimeOffset? LastFetchAt { get; init; }
/// <summary>
/// Additional cursor fields (string values) to merge.
/// </summary>
public IReadOnlyDictionary<string, string>? Additional { get; init; }
}
/// <summary>
/// Seeding specification describing the source, documents, and cursor edits to apply.
/// </summary>
public sealed record SourceStateSeedSpecification
{
/// <summary>
/// Source/connector name (e.g. <c>vndr.msrc</c>).
/// </summary>
public string Source { get; init; } = string.Empty;
/// <summary>
/// Documents that should be inserted or replaced before the cursor update.
/// </summary>
public IReadOnlyList<SourceStateSeedDocument> Documents { get; init; } = Array.Empty<SourceStateSeedDocument>();
/// <summary>
/// Cursor adjustments applied after documents are persisted.
/// </summary>
public SourceStateSeedCursor? Cursor { get; init; }
/// <summary>
/// Connector-level known advisory identifiers to merge into the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Optional completion timestamp. Defaults to the processor's time provider.
/// </summary>
public DateTimeOffset? CompletedAt { get; init; }
}
/// <summary>
/// Result returned after seeding completes.
/// </summary>
public sealed record SourceStateSeedResult(
int DocumentsProcessed,
int PendingDocumentsAdded,
int PendingMappingsAdded,
IReadOnlyCollection<Guid> DocumentIds,
IReadOnlyCollection<Guid> PendingDocumentIds,
IReadOnlyCollection<Guid> PendingMappingIds,
IReadOnlyCollection<string> KnownAdvisoriesAdded,
DateTimeOffset CompletedAt);
using StellaOps.Concelier.Connector.Common;
namespace StellaOps.Concelier.Connector.Common.State;
/// <summary>
/// Describes a raw upstream document that should be persisted for a connector during seeding.
/// </summary>
public sealed record SourceStateSeedDocument
{
/// <summary>
/// Absolute source URI. Must match the connector's upstream document identifier.
/// </summary>
public string Uri { get; init; } = string.Empty;
/// <summary>
/// Raw document payload. Required when creating or replacing a document.
/// </summary>
public byte[] Content { get; init; } = Array.Empty<byte>();
/// <summary>
/// Optional explicit document identifier. When provided it overrides auto-generated IDs.
/// </summary>
public Guid? DocumentId { get; init; }
/// <summary>
/// MIME type for the document payload.
/// </summary>
public string? ContentType { get; init; }
/// <summary>
/// Status assigned to the document. Defaults to <see cref="DocumentStatuses.PendingParse"/>.
/// </summary>
public string Status { get; init; } = DocumentStatuses.PendingParse;
/// <summary>
/// Optional HTTP-style headers persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Headers { get; init; }
/// <summary>
/// Source metadata (connector specific) persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
/// <summary>
/// Upstream ETag value, if available.
/// </summary>
public string? Etag { get; init; }
/// <summary>
/// Upstream last-modified timestamp, if available.
/// </summary>
public DateTimeOffset? LastModified { get; init; }
/// <summary>
/// Optional document expiration. When set a TTL will purge the raw payload after the configured retention.
/// </summary>
public DateTimeOffset? ExpiresAt { get; init; }
/// <summary>
/// Fetch timestamp stamped onto the document. Defaults to the seed completion timestamp.
/// </summary>
public DateTimeOffset? FetchedAt { get; init; }
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingDocuments</c> set.
/// </summary>
public bool AddToPendingDocuments { get; init; } = true;
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingMappings</c> set.
/// </summary>
public bool AddToPendingMappings { get; init; }
/// <summary>
/// Optional identifiers that should be recorded on the cursor to avoid duplicate ingestion.
/// </summary>
public IReadOnlyCollection<string>? KnownIdentifiers { get; init; }
}
/// <summary>
/// Cursor updates that should accompany seeded documents.
/// </summary>
public sealed record SourceStateSeedCursor
{
/// <summary>
/// Optional <c>pendingDocuments</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingDocuments { get; init; }
/// <summary>
/// Optional <c>pendingMappings</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingMappings { get; init; }
/// <summary>
/// Optional known advisory identifiers to merge with the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Upstream window watermark tracked by connectors that rely on last-modified cursors.
/// </summary>
public DateTimeOffset? LastModifiedCursor { get; init; }
/// <summary>
/// Optional fetch timestamp used by connectors that track the last polling instant.
/// </summary>
public DateTimeOffset? LastFetchAt { get; init; }
/// <summary>
/// Additional cursor fields (string values) to merge.
/// </summary>
public IReadOnlyDictionary<string, string>? Additional { get; init; }
}
/// <summary>
/// Seeding specification describing the source, documents, and cursor edits to apply.
/// </summary>
public sealed record SourceStateSeedSpecification
{
/// <summary>
/// Source/connector name (e.g. <c>vndr.msrc</c>).
/// </summary>
public string Source { get; init; } = string.Empty;
/// <summary>
/// Documents that should be inserted or replaced before the cursor update.
/// </summary>
public IReadOnlyList<SourceStateSeedDocument> Documents { get; init; } = Array.Empty<SourceStateSeedDocument>();
/// <summary>
/// Cursor adjustments applied after documents are persisted.
/// </summary>
public SourceStateSeedCursor? Cursor { get; init; }
/// <summary>
/// Connector-level known advisory identifiers to merge into the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Optional completion timestamp. Defaults to the processor's time provider.
/// </summary>
public DateTimeOffset? CompletedAt { get; init; }
}
/// <summary>
/// Result returned after seeding completes.
/// </summary>
public sealed record SourceStateSeedResult(
int DocumentsProcessed,
int PendingDocumentsAdded,
int PendingMappingsAdded,
IReadOnlyCollection<Guid> DocumentIds,
IReadOnlyCollection<Guid> PendingDocumentIds,
IReadOnlyCollection<Guid> PendingMappingIds,
IReadOnlyCollection<string> KnownAdvisoriesAdded,
DateTimeOffset CompletedAt);

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs
View File

@@ -1,6 +1,6 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common.Fetch;
using MongoContracts = StellaOps.Concelier.Storage;
using StellaOps.Cryptography;
@@ -62,7 +62,7 @@ public sealed class SourceStateSeedProcessor
}
var state = await _stateRepository.TryGetAsync(specification.Source, cancellationToken).ConfigureAwait(false);
var cursor = state?.Cursor ?? new BsonDocument();
var cursor = state?.Cursor ?? new DocumentObject();
var newlyPendingDocuments = MergeGuidArray(cursor, "pendingDocuments", pendingDocumentIds);
var newlyPendingMappings = MergeGuidArray(cursor, "pendingMappings", pendingMappingIds);
@@ -216,14 +216,14 @@ public sealed class SourceStateSeedProcessor
return new Dictionary<string, string>(values, StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyCollection<Guid> MergeGuidArray(BsonDocument cursor, string field, IReadOnlyCollection<Guid> additions)
private static IReadOnlyCollection<Guid> MergeGuidArray(DocumentObject cursor, string field, IReadOnlyCollection<Guid> additions)
{
if (additions.Count == 0)
{
return Array.Empty<Guid>();
}
var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray
var existing = cursor.TryGetValue(field, out var value) && value is DocumentArray existingArray
? existingArray.Select(AsGuid).Where(static g => g != Guid.Empty).ToHashSet()
: new HashSet<Guid>();
@@ -243,7 +243,7 @@ public sealed class SourceStateSeedProcessor
if (existing.Count > 0)
{
cursor[field] = new BsonArray(existing
cursor[field] = new DocumentArray(existing
.Select(static g => g.ToString("D"))
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
}
@@ -251,14 +251,14 @@ public sealed class SourceStateSeedProcessor
return newlyAdded.AsReadOnly();
}
private static IReadOnlyCollection<string> MergeStringArray(BsonDocument cursor, string field, IReadOnlyCollection<string> additions)
private static IReadOnlyCollection<string> MergeStringArray(DocumentObject cursor, string field, IReadOnlyCollection<string> additions)
{
if (additions.Count == 0)
{
return Array.Empty<string>();
}
var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray
var existing = cursor.TryGetValue(field, out var value) && value is DocumentArray existingArray
? existingArray.Select(static v => v?.AsString ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.ToHashSet(StringComparer.OrdinalIgnoreCase)
@@ -281,14 +281,14 @@ public sealed class SourceStateSeedProcessor
if (existing.Count > 0)
{
cursor[field] = new BsonArray(existing
cursor[field] = new DocumentArray(existing
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
}
return newlyAdded.AsReadOnly();
}
private static Guid AsGuid(BsonValue value)
private static Guid AsGuid(DocumentValue value)
{
if (value is null)
{

214
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Telemetry/SourceDiagnostics.cs
View File

@@ -1,107 +1,107 @@
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Telemetry;
/// <summary>
/// Central telemetry instrumentation for connector HTTP operations.
/// </summary>
public static class SourceDiagnostics
{
public const string ActivitySourceName = "StellaOps.Concelier.Connector";
public const string MeterName = "StellaOps.Concelier.Connector";
private static readonly ActivitySource ActivitySource = new(ActivitySourceName);
private static readonly Meter Meter = new(MeterName);
private static readonly Counter<long> HttpRequestCounter = Meter.CreateCounter<long>("concelier.source.http.requests");
private static readonly Counter<long> HttpRetryCounter = Meter.CreateCounter<long>("concelier.source.http.retries");
private static readonly Counter<long> HttpFailureCounter = Meter.CreateCounter<long>("concelier.source.http.failures");
private static readonly Counter<long> HttpNotModifiedCounter = Meter.CreateCounter<long>("concelier.source.http.not_modified");
private static readonly Histogram<double> HttpDuration = Meter.CreateHistogram<double>("concelier.source.http.duration", unit: "ms");
private static readonly Histogram<long> HttpPayloadBytes = Meter.CreateHistogram<long>("concelier.source.http.payload_bytes", unit: "byte");
public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName)
{
var tags = new ActivityTagsCollection
{
{ "concelier.source", sourceName },
{ "http.method", httpMethod },
{ "http.url", requestUri.ToString() },
};
if (!string.IsNullOrWhiteSpace(clientName))
{
tags.Add("http.client_name", clientName!);
}
return ActivitySource.StartActivity("SourceFetch", ActivityKind.Client, parentContext: default, tags: tags);
}
public static void RecordHttpRequest(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount, TimeSpan duration, long? contentLength, string? rateLimitRemaining)
{
var tags = BuildDefaultTags(sourceName, clientName, statusCode, attemptCount);
HttpRequestCounter.Add(1, tags);
HttpDuration.Record(duration.TotalMilliseconds, tags);
if (contentLength.HasValue && contentLength.Value >= 0)
{
HttpPayloadBytes.Record(contentLength.Value, tags);
}
if (statusCode == HttpStatusCode.NotModified)
{
HttpNotModifiedCounter.Add(1, tags);
}
if ((int)statusCode >= 500 || statusCode == HttpStatusCode.TooManyRequests)
{
HttpFailureCounter.Add(1, tags);
}
if (!string.IsNullOrWhiteSpace(rateLimitRemaining) && long.TryParse(rateLimitRemaining, out var remaining))
{
tags.Add("http.rate_limit.remaining", remaining);
}
}
public static void RecordRetry(string sourceName, string? clientName, HttpStatusCode? statusCode, int attempt, TimeSpan delay)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.retry_attempt", attempt },
{ "http.retry_delay_ms", delay.TotalMilliseconds },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
if (statusCode.HasValue)
{
tags.Add("http.status_code", (int)statusCode.Value);
}
HttpRetryCounter.Add(1, tags);
}
private static TagList BuildDefaultTags(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.status_code", (int)statusCode },
{ "http.attempts", attemptCount },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
return tags;
}
}
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Telemetry;
/// <summary>
/// Central telemetry instrumentation for connector HTTP operations.
/// </summary>
public static class SourceDiagnostics
{
public const string ActivitySourceName = "StellaOps.Concelier.Connector";
public const string MeterName = "StellaOps.Concelier.Connector";
private static readonly ActivitySource ActivitySource = new(ActivitySourceName);
private static readonly Meter Meter = new(MeterName);
private static readonly Counter<long> HttpRequestCounter = Meter.CreateCounter<long>("concelier.source.http.requests");
private static readonly Counter<long> HttpRetryCounter = Meter.CreateCounter<long>("concelier.source.http.retries");
private static readonly Counter<long> HttpFailureCounter = Meter.CreateCounter<long>("concelier.source.http.failures");
private static readonly Counter<long> HttpNotModifiedCounter = Meter.CreateCounter<long>("concelier.source.http.not_modified");
private static readonly Histogram<double> HttpDuration = Meter.CreateHistogram<double>("concelier.source.http.duration", unit: "ms");
private static readonly Histogram<long> HttpPayloadBytes = Meter.CreateHistogram<long>("concelier.source.http.payload_bytes", unit: "byte");
public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName)
{
var tags = new ActivityTagsCollection
{
{ "concelier.source", sourceName },
{ "http.method", httpMethod },
{ "http.url", requestUri.ToString() },
};
if (!string.IsNullOrWhiteSpace(clientName))
{
tags.Add("http.client_name", clientName!);
}
return ActivitySource.StartActivity("SourceFetch", ActivityKind.Client, parentContext: default, tags: tags);
}
public static void RecordHttpRequest(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount, TimeSpan duration, long? contentLength, string? rateLimitRemaining)
{
var tags = BuildDefaultTags(sourceName, clientName, statusCode, attemptCount);
HttpRequestCounter.Add(1, tags);
HttpDuration.Record(duration.TotalMilliseconds, tags);
if (contentLength.HasValue && contentLength.Value >= 0)
{
HttpPayloadBytes.Record(contentLength.Value, tags);
}
if (statusCode == HttpStatusCode.NotModified)
{
HttpNotModifiedCounter.Add(1, tags);
}
if ((int)statusCode >= 500 || statusCode == HttpStatusCode.TooManyRequests)
{
HttpFailureCounter.Add(1, tags);
}
if (!string.IsNullOrWhiteSpace(rateLimitRemaining) && long.TryParse(rateLimitRemaining, out var remaining))
{
tags.Add("http.rate_limit.remaining", remaining);
}
}
public static void RecordRetry(string sourceName, string? clientName, HttpStatusCode? statusCode, int attempt, TimeSpan delay)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.retry_attempt", attempt },
{ "http.retry_delay_ms", delay.TotalMilliseconds },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
if (statusCode.HasValue)
{
tags.Add("http.status_code", (int)statusCode.Value);
}
HttpRetryCounter.Add(1, tags);
}
private static TagList BuildDefaultTags(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.status_code", (int)statusCode },
{ "http.attempts", attemptCount },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
return tags;
}
}

420
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Testing/CannedHttpMessageHandler.cs
View File

@@ -1,210 +1,210 @@
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http;
using System.Text;
namespace StellaOps.Concelier.Connector.Common.Testing;
/// <summary>
/// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method.
/// Tracks requests for assertions and supports fallbacks/exceptions.
/// </summary>
public sealed class CannedHttpMessageHandler : HttpMessageHandler
{
private readonly ConcurrentDictionary<RequestKey, ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>> _responses =
new(RequestKeyComparer.Instance);
private readonly ConcurrentQueue<CannedRequestRecord> _requests = new();
private Func<HttpRequestMessage, HttpResponseMessage>? _fallback;
/// <summary>
/// Recorded requests in arrival order.
/// </summary>
public IReadOnlyCollection<CannedRequestRecord> Requests => _requests.ToArray();
/// <summary>
/// Registers a canned response for a GET request to <paramref name="requestUri"/>.
/// </summary>
public void AddResponse(Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(HttpMethod.Get, requestUri, _ => factory());
/// <summary>
/// Registers a canned response for the specified method and URI.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(method, requestUri, _ => factory());
/// <summary>
/// Registers a canned response using the full request context.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpRequestMessage, HttpResponseMessage> factory)
{
ArgumentNullException.ThrowIfNull(method);
ArgumentNullException.ThrowIfNull(requestUri);
ArgumentNullException.ThrowIfNull(factory);
var key = new RequestKey(method, requestUri);
var queue = _responses.GetOrAdd(key, static _ => new ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>());
queue.Enqueue(factory);
}
/// <summary>
/// Registers an exception to be thrown for the specified request.
/// </summary>
public void AddException(HttpMethod method, Uri requestUri, Exception exception)
{
ArgumentNullException.ThrowIfNull(exception);
AddResponse(method, requestUri, _ => throw exception);
}
/// <summary>
/// Registers a fallback used when no specific response is queued for a request.
/// </summary>
public void SetFallback(Func<HttpRequestMessage, HttpResponseMessage> fallback)
{
ArgumentNullException.ThrowIfNull(fallback);
_fallback = fallback;
}
/// <summary>
/// Clears registered responses and captured requests.
/// </summary>
public void Clear()
{
_responses.Clear();
while (_requests.TryDequeue(out _))
{
}
_fallback = null;
}
/// <summary>
/// Throws if any responses remain queued.
/// </summary>
public void AssertNoPendingResponses()
{
foreach (var queue in _responses.Values)
{
if (!queue.IsEmpty)
{
throw new InvalidOperationException("Not all canned responses were consumed.");
}
}
}
/// <summary>
/// Creates an <see cref="HttpClient"/> wired to this handler.
/// </summary>
public HttpClient CreateClient()
=> new(this, disposeHandler: false)
{
Timeout = TimeSpan.FromSeconds(10),
};
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri is null)
{
throw new InvalidOperationException("Request URI is required for canned responses.");
}
var key = new RequestKey(request.Method ?? HttpMethod.Get, request.RequestUri);
var factory = DequeueFactory(key);
if (factory is null)
{
if (_fallback is null)
{
throw new InvalidOperationException($"No canned response registered for {request.Method} {request.RequestUri}.");
}
factory = _fallback;
}
var snapshot = CaptureRequest(request);
_requests.Enqueue(snapshot);
var response = factory(request);
response.RequestMessage ??= request;
return Task.FromResult(response);
}
private Func<HttpRequestMessage, HttpResponseMessage>? DequeueFactory(RequestKey key)
{
if (_responses.TryGetValue(key, out var queue) && queue.TryDequeue(out var factory))
{
return factory;
}
return null;
}
private static CannedRequestRecord CaptureRequest(HttpRequestMessage request)
{
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var header in request.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
if (request.Content is not null)
{
foreach (var header in request.Content.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
}
return new CannedRequestRecord(
Timestamp: DateTimeOffset.UtcNow,
Method: request.Method ?? HttpMethod.Get,
Uri: request.RequestUri!,
Headers: headers);
}
private readonly record struct RequestKey(HttpMethod Method, string Uri)
{
public RequestKey(HttpMethod method, Uri uri)
: this(method, uri.ToString())
{
}
public bool Equals(RequestKey other)
=> string.Equals(Method.Method, other.Method.Method, StringComparison.OrdinalIgnoreCase)
&& string.Equals(Uri, other.Uri, StringComparison.OrdinalIgnoreCase);
public override int GetHashCode()
{
var methodHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Method.Method);
var uriHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Uri);
return HashCode.Combine(methodHash, uriHash);
}
}
private sealed class RequestKeyComparer : IEqualityComparer<RequestKey>
{
public static readonly RequestKeyComparer Instance = new();
public bool Equals(RequestKey x, RequestKey y) => x.Equals(y);
public int GetHashCode(RequestKey obj) => obj.GetHashCode();
}
public readonly record struct CannedRequestRecord(DateTimeOffset Timestamp, HttpMethod Method, Uri Uri, IReadOnlyDictionary<string, string> Headers);
private static HttpResponseMessage BuildTextResponse(HttpStatusCode statusCode, string content, string contentType)
{
var message = new HttpResponseMessage(statusCode)
{
Content = new StringContent(content, Encoding.UTF8, contentType),
};
return message;
}
public void AddJsonResponse(Uri requestUri, string json, HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, json, "application/json"));
public void AddTextResponse(Uri requestUri, string content, string contentType = "text/plain", HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, content, contentType));
}
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http;
using System.Text;
namespace StellaOps.Concelier.Connector.Common.Testing;
/// <summary>
/// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method.
/// Tracks requests for assertions and supports fallbacks/exceptions.
/// </summary>
public sealed class CannedHttpMessageHandler : HttpMessageHandler
{
private readonly ConcurrentDictionary<RequestKey, ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>> _responses =
new(RequestKeyComparer.Instance);
private readonly ConcurrentQueue<CannedRequestRecord> _requests = new();
private Func<HttpRequestMessage, HttpResponseMessage>? _fallback;
/// <summary>
/// Recorded requests in arrival order.
/// </summary>
public IReadOnlyCollection<CannedRequestRecord> Requests => _requests.ToArray();
/// <summary>
/// Registers a canned response for a GET request to <paramref name="requestUri"/>.
/// </summary>
public void AddResponse(Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(HttpMethod.Get, requestUri, _ => factory());
/// <summary>
/// Registers a canned response for the specified method and URI.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(method, requestUri, _ => factory());
/// <summary>
/// Registers a canned response using the full request context.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpRequestMessage, HttpResponseMessage> factory)
{
ArgumentNullException.ThrowIfNull(method);
ArgumentNullException.ThrowIfNull(requestUri);
ArgumentNullException.ThrowIfNull(factory);
var key = new RequestKey(method, requestUri);
var queue = _responses.GetOrAdd(key, static _ => new ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>());
queue.Enqueue(factory);
}
/// <summary>
/// Registers an exception to be thrown for the specified request.
/// </summary>
public void AddException(HttpMethod method, Uri requestUri, Exception exception)
{
ArgumentNullException.ThrowIfNull(exception);
AddResponse(method, requestUri, _ => throw exception);
}
/// <summary>
/// Registers a fallback used when no specific response is queued for a request.
/// </summary>
public void SetFallback(Func<HttpRequestMessage, HttpResponseMessage> fallback)
{
ArgumentNullException.ThrowIfNull(fallback);
_fallback = fallback;
}
/// <summary>
/// Clears registered responses and captured requests.
/// </summary>
public void Clear()
{
_responses.Clear();
while (_requests.TryDequeue(out _))
{
}
_fallback = null;
}
/// <summary>
/// Throws if any responses remain queued.
/// </summary>
public void AssertNoPendingResponses()
{
foreach (var queue in _responses.Values)
{
if (!queue.IsEmpty)
{
throw new InvalidOperationException("Not all canned responses were consumed.");
}
}
}
/// <summary>
/// Creates an <see cref="HttpClient"/> wired to this handler.
/// </summary>
public HttpClient CreateClient()
=> new(this, disposeHandler: false)
{
Timeout = TimeSpan.FromSeconds(10),
};
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri is null)
{
throw new InvalidOperationException("Request URI is required for canned responses.");
}
var key = new RequestKey(request.Method ?? HttpMethod.Get, request.RequestUri);
var factory = DequeueFactory(key);
if (factory is null)
{
if (_fallback is null)
{
throw new InvalidOperationException($"No canned response registered for {request.Method} {request.RequestUri}.");
}
factory = _fallback;
}
var snapshot = CaptureRequest(request);
_requests.Enqueue(snapshot);
var response = factory(request);
response.RequestMessage ??= request;
return Task.FromResult(response);
}
private Func<HttpRequestMessage, HttpResponseMessage>? DequeueFactory(RequestKey key)
{
if (_responses.TryGetValue(key, out var queue) && queue.TryDequeue(out var factory))
{
return factory;
}
return null;
}
private static CannedRequestRecord CaptureRequest(HttpRequestMessage request)
{
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var header in request.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
if (request.Content is not null)
{
foreach (var header in request.Content.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
}
return new CannedRequestRecord(
Timestamp: DateTimeOffset.UtcNow,
Method: request.Method ?? HttpMethod.Get,
Uri: request.RequestUri!,
Headers: headers);
}
private readonly record struct RequestKey(HttpMethod Method, string Uri)
{
public RequestKey(HttpMethod method, Uri uri)
: this(method, uri.ToString())
{
}
public bool Equals(RequestKey other)
=> string.Equals(Method.Method, other.Method.Method, StringComparison.OrdinalIgnoreCase)
&& string.Equals(Uri, other.Uri, StringComparison.OrdinalIgnoreCase);
public override int GetHashCode()
{
var methodHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Method.Method);
var uriHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Uri);
return HashCode.Combine(methodHash, uriHash);
}
}
private sealed class RequestKeyComparer : IEqualityComparer<RequestKey>
{
public static readonly RequestKeyComparer Instance = new();
public bool Equals(RequestKey x, RequestKey y) => x.Equals(y);
public int GetHashCode(RequestKey obj) => obj.GetHashCode();
}
public readonly record struct CannedRequestRecord(DateTimeOffset Timestamp, HttpMethod Method, Uri Uri, IReadOnlyDictionary<string, string> Headers);
private static HttpResponseMessage BuildTextResponse(HttpStatusCode statusCode, string content, string contentType)
{
var message = new HttpResponseMessage(statusCode)
{
Content = new StringContent(content, Encoding.UTF8, contentType),
};
return message;
}
public void AddJsonResponse(Uri requestUri, string json, HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, json, "application/json"));
public void AddTextResponse(Uri requestUri, string content, string contentType = "text/plain", HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, content, contentType));
}

124
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Url/UrlNormalizer.cs
View File

@@ -1,62 +1,62 @@
namespace StellaOps.Concelier.Connector.Common.Url;
/// <summary>
/// Utilities for normalizing URLs from upstream feeds.
/// </summary>
public static class UrlNormalizer
{
/// <summary>
/// Attempts to normalize <paramref name="value"/> relative to <paramref name="baseUri"/>.
/// Removes fragments and enforces HTTPS when possible.
/// </summary>
public static bool TryNormalize(string? value, Uri? baseUri, out Uri? normalized, bool stripFragment = true, bool forceHttps = false)
{
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!Uri.TryCreate(value.Trim(), UriKind.RelativeOrAbsolute, out var candidate))
{
return false;
}
if (!candidate.IsAbsoluteUri)
{
if (baseUri is null)
{
return false;
}
if (!Uri.TryCreate(baseUri, candidate, out candidate))
{
return false;
}
}
if (forceHttps && string.Equals(candidate.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase))
{
candidate = new UriBuilder(candidate) { Scheme = Uri.UriSchemeHttps, Port = candidate.IsDefaultPort ? -1 : candidate.Port }.Uri;
}
if (stripFragment && !string.IsNullOrEmpty(candidate.Fragment))
{
var builder = new UriBuilder(candidate) { Fragment = string.Empty };
candidate = builder.Uri;
}
normalized = candidate;
return true;
}
public static Uri NormalizeOrThrow(string value, Uri? baseUri = null, bool stripFragment = true, bool forceHttps = false)
{
if (!TryNormalize(value, baseUri, out var normalized, stripFragment, forceHttps) || normalized is null)
{
throw new FormatException($"Value '{value}' is not a valid URI");
}
return normalized;
}
}
namespace StellaOps.Concelier.Connector.Common.Url;
/// <summary>
/// Utilities for normalizing URLs from upstream feeds.
/// </summary>
public static class UrlNormalizer
{
/// <summary>
/// Attempts to normalize <paramref name="value"/> relative to <paramref name="baseUri"/>.
/// Removes fragments and enforces HTTPS when possible.
/// </summary>
public static bool TryNormalize(string? value, Uri? baseUri, out Uri? normalized, bool stripFragment = true, bool forceHttps = false)
{
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!Uri.TryCreate(value.Trim(), UriKind.RelativeOrAbsolute, out var candidate))
{
return false;
}
if (!candidate.IsAbsoluteUri)
{
if (baseUri is null)
{
return false;
}
if (!Uri.TryCreate(baseUri, candidate, out candidate))
{
return false;
}
}
if (forceHttps && string.Equals(candidate.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase))
{
candidate = new UriBuilder(candidate) { Scheme = Uri.UriSchemeHttps, Port = candidate.IsDefaultPort ? -1 : candidate.Port }.Uri;
}
if (stripFragment && !string.IsNullOrEmpty(candidate.Fragment))
{
var builder = new UriBuilder(candidate) { Fragment = string.Empty };
candidate = builder.Uri;
}
normalized = candidate;
return true;
}
public static Uri NormalizeOrThrow(string value, Uri? baseUri = null, bool stripFragment = true, bool forceHttps = false)
{
if (!TryNormalize(value, baseUri, out var normalized, stripFragment, forceHttps) || normalized is null)
{
throw new FormatException($"Value '{value}' is not a valid URI");
}
return normalized;
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Xml/IXmlSchemaValidator.cs
View File

@@ -1,9 +1,9 @@
using System.Xml.Linq;
using System.Xml.Schema;
namespace StellaOps.Concelier.Connector.Common.Xml;
public interface IXmlSchemaValidator
{
void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName);
}
using System.Xml.Linq;
using System.Xml.Schema;
namespace StellaOps.Concelier.Connector.Common.Xml;
public interface IXmlSchemaValidator
{
void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName);
}

Some files were not shown because too many files have changed in this diff Show More