stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/PaginationPlanner.cs
View File

@@ -1,29 +1,29 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
/// </summary>
public static class PaginationPlanner
{
/// <summary>
/// Enumerates additional page start indices given the total result count returned by the source.
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
/// </summary>
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
{
if (totalResults <= 0 || resultsPerPage <= 0)
{
yield break;
}
if (firstPageStartIndex < 0)
{
firstPageStartIndex = 0;
}
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
{
yield return start;
}
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Provides helpers for computing pagination start indices for sources that expose total result counts.
/// </summary>
public static class PaginationPlanner
{
/// <summary>
/// Enumerates additional page start indices given the total result count returned by the source.
/// The first page (at <paramref name="firstPageStartIndex"/>) is assumed to be already fetched.
/// </summary>
public static IEnumerable<int> EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0)
{
if (totalResults <= 0 || resultsPerPage <= 0)
{
yield break;
}
if (firstPageStartIndex < 0)
{
firstPageStartIndex = 0;
}
for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage)
{
yield return start;
}
}
}

86
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorOptions.cs
View File

@@ -1,43 +1,43 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Configuration applied when advancing sliding time-window cursors.
/// </summary>
public sealed class TimeWindowCursorOptions
{
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
public void EnsureValid()
{
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Window overlap cannot be negative.");
}
if (Overlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill must be positive.");
}
if (MinimumWindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Minimum window size must be positive.");
}
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Configuration applied when advancing sliding time-window cursors.
/// </summary>
public sealed class TimeWindowCursorOptions
{
public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4);
public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7);
public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1);
public void EnsureValid()
{
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Window overlap cannot be negative.");
}
if (Overlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill must be positive.");
}
if (MinimumWindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Minimum window size must be positive.");
}
}
}

100
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorPlanner.cs
View File

@@ -1,50 +1,50 @@
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Utility methods for computing sliding time-window ranges used by connectors.
/// </summary>
public static class TimeWindowCursorPlanner
{
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
{
ArgumentNullException.ThrowIfNull(options);
options.EnsureValid();
var effectiveState = state ?? TimeWindowCursorState.Empty;
var earliest = now - options.InitialBackfill;
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
if (anchorEnd < earliest)
{
anchorEnd = earliest;
}
var start = anchorEnd - options.Overlap;
if (start < earliest)
{
start = earliest;
}
var end = start + options.WindowSize;
if (end > now)
{
end = now;
}
if (end <= start)
{
end = start + options.MinimumWindowSize;
if (end > now)
{
end = now;
}
}
if (end <= start)
{
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
}
return new TimeWindow(start, end);
}
}
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Utility methods for computing sliding time-window ranges used by connectors.
/// </summary>
public static class TimeWindowCursorPlanner
{
public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options)
{
ArgumentNullException.ThrowIfNull(options);
options.EnsureValid();
var effectiveState = state ?? TimeWindowCursorState.Empty;
var earliest = now - options.InitialBackfill;
var anchorEnd = effectiveState.LastWindowEnd ?? earliest;
if (anchorEnd < earliest)
{
anchorEnd = earliest;
}
var start = anchorEnd - options.Overlap;
if (start < earliest)
{
start = earliest;
}
var end = start + options.WindowSize;
if (end > now)
{
end = now;
}
if (end <= start)
{
end = start + options.MinimumWindowSize;
if (end > now)
{
end = now;
}
}
if (end <= start)
{
throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options.");
}
return new TimeWindow(start, end);
}
}

168
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs
View File

@@ -1,84 +1,84 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Represents the persisted state of a sliding time-window cursor.
/// </summary>
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
{
public static TimeWindowCursorState Empty { get; } = new(null, null);
public TimeWindowCursorState WithWindow(TimeWindow window)
{
return new TimeWindowCursorState(window.Start, window.End);
}
public BsonDocument ToBsonDocument(string startField = "windowStart", string endField = "windowEnd")
{
var document = new BsonDocument();
WriteTo(document, startField, endField);
return document;
}
public void WriteTo(BsonDocument document, string startField = "windowStart", string endField = "windowEnd")
{
ArgumentNullException.ThrowIfNull(document);
ArgumentException.ThrowIfNullOrEmpty(startField);
ArgumentException.ThrowIfNullOrEmpty(endField);
document.Remove(startField);
document.Remove(endField);
if (LastWindowStart.HasValue)
{
document[startField] = LastWindowStart.Value.UtcDateTime;
}
if (LastWindowEnd.HasValue)
{
document[endField] = LastWindowEnd.Value.UtcDateTime;
}
}
public static TimeWindowCursorState FromBsonDocument(BsonDocument? document, string startField = "windowStart", string endField = "windowEnd")
{
if (document is null)
{
return Empty;
}
DateTimeOffset? start = null;
DateTimeOffset? end = null;
if (document.TryGetValue(startField, out var startValue))
{
start = ReadDateTimeOffset(startValue);
}
if (document.TryGetValue(endField, out var endValue))
{
end = ReadDateTimeOffset(endValue);
}
return new TimeWindowCursorState(start, end);
}
private static DateTimeOffset? ReadDateTimeOffset(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
/// <summary>
/// Simple value object describing a time window.
/// </summary>
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
{
public TimeSpan Duration => End - Start;
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Common.Cursors;
/// <summary>
/// Represents the persisted state of a sliding time-window cursor.
/// </summary>
public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd)
{
public static TimeWindowCursorState Empty { get; } = new(null, null);
public TimeWindowCursorState WithWindow(TimeWindow window)
{
return new TimeWindowCursorState(window.Start, window.End);
}
public DocumentObject ToDocumentObject(string startField = "windowStart", string endField = "windowEnd")
{
var document = new DocumentObject();
WriteTo(document, startField, endField);
return document;
}
public void WriteTo(DocumentObject document, string startField = "windowStart", string endField = "windowEnd")
{
ArgumentNullException.ThrowIfNull(document);
ArgumentException.ThrowIfNullOrEmpty(startField);
ArgumentException.ThrowIfNullOrEmpty(endField);
document.Remove(startField);
document.Remove(endField);
if (LastWindowStart.HasValue)
{
document[startField] = LastWindowStart.Value.UtcDateTime;
}
if (LastWindowEnd.HasValue)
{
document[endField] = LastWindowEnd.Value.UtcDateTime;
}
}
public static TimeWindowCursorState FromDocumentObject(DocumentObject? document, string startField = "windowStart", string endField = "windowEnd")
{
if (document is null)
{
return Empty;
}
DateTimeOffset? start = null;
DateTimeOffset? end = null;
if (document.TryGetValue(startField, out var startValue))
{
start = ReadDateTimeOffset(startValue);
}
if (document.TryGetValue(endField, out var endValue))
{
end = ReadDateTimeOffset(endValue);
}
return new TimeWindowCursorState(start, end);
}
private static DateTimeOffset? ReadDateTimeOffset(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
/// <summary>
/// Simple value object describing a time window.
/// </summary>
public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End)
{
public TimeSpan Duration => End - Start;
}

54
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/DocumentStatuses.cs
View File

@@ -1,27 +1,27 @@
namespace StellaOps.Concelier.Connector.Common;
/// <summary>
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
/// </summary>
public static class DocumentStatuses
{
/// <summary>
/// Document captured from the upstream source and awaiting schema validation/parsing.
/// </summary>
public const string PendingParse = "pending-parse";
/// <summary>
/// Document parsed and sanitized; awaiting canonical mapping.
/// </summary>
public const string PendingMap = "pending-map";
/// <summary>
/// Document fully mapped to canonical advisories.
/// </summary>
public const string Mapped = "mapped";
/// <summary>
/// Document failed processing; requires manual intervention before retry.
/// </summary>
public const string Failed = "failed";
}
namespace StellaOps.Concelier.Connector.Common;
/// <summary>
/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages.
/// </summary>
public static class DocumentStatuses
{
/// <summary>
/// Document captured from the upstream source and awaiting schema validation/parsing.
/// </summary>
public const string PendingParse = "pending-parse";
/// <summary>
/// Document parsed and sanitized; awaiting canonical mapping.
/// </summary>
public const string PendingMap = "pending-map";
/// <summary>
/// Document fully mapped to canonical advisories.
/// </summary>
public const string Mapped = "mapped";
/// <summary>
/// Document failed processing; requires manual intervention before retry.
/// </summary>
public const string Failed = "failed";
}

86
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/CryptoJitterSource.cs
View File

@@ -1,43 +1,43 @@
using System.Security.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
/// </summary>
public sealed class CryptoJitterSource : IJitterSource
{
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
{
if (maxInclusive < minInclusive)
{
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
}
if (minInclusive < TimeSpan.Zero)
{
minInclusive = TimeSpan.Zero;
}
if (maxInclusive == minInclusive)
{
return minInclusive;
}
var minTicks = minInclusive.Ticks;
var maxTicks = maxInclusive.Ticks;
var range = maxTicks - minTicks;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer);
var ratio = sample / (double)ulong.MaxValue;
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
if (jitterTicks > range)
{
jitterTicks = range;
}
return TimeSpan.FromTicks(minTicks + jitterTicks);
}
}
using System.Security.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
/// </summary>
public sealed class CryptoJitterSource : IJitterSource
{
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
{
if (maxInclusive < minInclusive)
{
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
}
if (minInclusive < TimeSpan.Zero)
{
minInclusive = TimeSpan.Zero;
}
if (maxInclusive == minInclusive)
{
return minInclusive;
}
var minTicks = minInclusive.Ticks;
var maxTicks = maxInclusive.Ticks;
var range = maxTicks - minTicks;
Span<byte> buffer = stackalloc byte[8];
RandomNumberGenerator.Fill(buffer);
var sample = BitConverter.ToUInt64(buffer);
var ratio = sample / (double)ulong.MaxValue;
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
if (jitterTicks > range)
{
jitterTicks = range;
}
return TimeSpan.FromTicks(minTicks + jitterTicks);
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/IJitterSource.cs
View File

@@ -1,9 +1,9 @@
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Produces random jitter durations used to decorrelate retries.
/// </summary>
public interface IJitterSource
{
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
}
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Produces random jitter durations used to decorrelate retries.
/// </summary>
public interface IJitterSource
{
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
}

46
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchContentResult.cs
View File

@@ -1,18 +1,18 @@
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Result of fetching raw response content without persisting a document.
/// </summary>
public sealed record SourceFetchContentResult
{
private SourceFetchContentResult(
HttpStatusCode statusCode,
byte[]? content,
bool notModified,
string? etag,
DateTimeOffset? lastModified,
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Result of fetching raw response content without persisting a document.
/// </summary>
public sealed record SourceFetchContentResult
{
private SourceFetchContentResult(
HttpStatusCode statusCode,
byte[]? content,
bool notModified,
string? etag,
DateTimeOffset? lastModified,
string? contentType,
int attempts,
IReadOnlyDictionary<string, string>? headers)
@@ -30,14 +30,14 @@ public sealed record SourceFetchContentResult
public HttpStatusCode StatusCode { get; }
public byte[]? Content { get; }
public bool IsSuccess => Content is not null;
public bool IsNotModified { get; }
public string? ETag { get; }
public DateTimeOffset? LastModified { get; }
public bool IsSuccess => Content is not null;
public bool IsNotModified { get; }
public string? ETag { get; }
public DateTimeOffset? LastModified { get; }
public string? ContentType { get; }

48
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchRequest.cs
View File

@@ -1,24 +1,24 @@
using System.Collections.Generic;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Parameters describing a fetch operation for a source connector.
/// </summary>
public sealed record SourceFetchRequest(
string ClientName,
string SourceName,
HttpMethod Method,
Uri RequestUri,
IReadOnlyDictionary<string, string>? Metadata = null,
string? ETag = null,
DateTimeOffset? LastModified = null,
TimeSpan? TimeoutOverride = null,
IReadOnlyList<string>? AcceptHeaders = null)
{
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
: this(clientName, sourceName, HttpMethod.Get, requestUri)
{
}
}
using System.Collections.Generic;
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Parameters describing a fetch operation for a source connector.
/// </summary>
public sealed record SourceFetchRequest(
string ClientName,
string SourceName,
HttpMethod Method,
Uri RequestUri,
IReadOnlyDictionary<string, string>? Metadata = null,
string? ETag = null,
DateTimeOffset? LastModified = null,
TimeSpan? TimeoutOverride = null,
IReadOnlyList<string>? AcceptHeaders = null)
{
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
: this(clientName, sourceName, HttpMethod.Get, requestUri)
{
}
}

68
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchResult.cs
View File

@@ -1,34 +1,34 @@
using System.Net;
using StellaOps.Concelier.Storage.Contracts;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Outcome of fetching a raw document from an upstream source.
/// </summary>
public sealed record SourceFetchResult
{
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
{
StatusCode = statusCode;
Document = document;
IsNotModified = notModified;
}
public HttpStatusCode StatusCode { get; }
public StorageDocument? Document { get; }
public bool IsSuccess => Document is not null;
public bool IsNotModified { get; }
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
=> new(statusCode, document, notModified: false);
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: true);
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: false);
}
using System.Net;
using StellaOps.Concelier.Storage.Contracts;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Outcome of fetching a raw document from an upstream source.
/// </summary>
public sealed record SourceFetchResult
{
private SourceFetchResult(HttpStatusCode statusCode, StorageDocument? document, bool notModified)
{
StatusCode = statusCode;
Document = document;
IsNotModified = notModified;
}
public HttpStatusCode StatusCode { get; }
public StorageDocument? Document { get; }
public bool IsSuccess => Document is not null;
public bool IsNotModified { get; }
public static SourceFetchResult Success(StorageDocument document, HttpStatusCode statusCode)
=> new(statusCode, document, notModified: false);
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: true);
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
=> new(statusCode, null, notModified: false);
}

1602
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs
View File

File diff suppressed because it is too large Load Diff

88
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceRetryPolicy.cs
View File

@@ -2,10 +2,10 @@ using System.Globalization;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Fetch;
/// <summary>
/// Provides retry/backoff behavior for source HTTP fetches.
/// </summary>
/// <summary>
/// Provides retry/backoff behavior for source HTTP fetches.
/// </summary>
internal static class SourceRetryPolicy
{
private static readonly StringComparer HeaderComparer = StringComparer.OrdinalIgnoreCase;
@@ -15,34 +15,34 @@ internal static class SourceRetryPolicy
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> sender,
int maxAttempts,
TimeSpan baseDelay,
IJitterSource jitterSource,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
var attempt = 0;
while (true)
{
attempt++;
using var request = requestFactory();
HttpResponseMessage response;
try
{
response = await sender(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (attempt < maxAttempts)
{
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
IJitterSource jitterSource,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
var attempt = 0;
while (true)
{
attempt++;
using var request = requestFactory();
HttpResponseMessage response;
try
{
response = await sender(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (attempt < maxAttempts)
{
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
if (NeedsRetry(response) && attempt < maxAttempts)
{
var delay = ComputeDelay(
@@ -55,11 +55,11 @@ internal static class SourceRetryPolicy
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
continue;
}
return response;
}
}
return response;
}
}
private static bool NeedsRetry(HttpResponseMessage response)
{
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
@@ -76,13 +76,13 @@ internal static class SourceRetryPolicy
return status >= 500 && status < 600;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
return retryAfter.Value;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
return retryAfter.Value;
}
var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1));
var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250))
?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250));

308
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Html/HtmlContentSanitizer.cs
View File

@@ -1,79 +1,79 @@
using System.Linq;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Url;
namespace StellaOps.Concelier.Connector.Common.Html;
/// <summary>
/// Sanitizes untrusted HTML fragments produced by upstream advisories.
/// Removes executable content, enforces an allowlist of elements, and normalizes anchor href values.
/// </summary>
public sealed class HtmlContentSanitizer
{
private static readonly HashSet<string> AllowedElements = new(StringComparer.OrdinalIgnoreCase)
using System.Linq;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Url;
namespace StellaOps.Concelier.Connector.Common.Html;
/// <summary>
/// Sanitizes untrusted HTML fragments produced by upstream advisories.
/// Removes executable content, enforces an allowlist of elements, and normalizes anchor href values.
/// </summary>
public sealed class HtmlContentSanitizer
{
private static readonly HashSet<string> AllowedElements = new(StringComparer.OrdinalIgnoreCase)
{
"a", "abbr", "article", "b", "body", "blockquote", "br", "code", "dd", "div", "dl", "dt",
"em", "h1", "h2", "h3", "h4", "h5", "h6", "html", "i", "li", "ol", "p", "pre", "s",
"section", "small", "span", "strong", "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "ul"
};
private static readonly HashSet<string> UrlAttributes = new(StringComparer.OrdinalIgnoreCase)
{
"href", "src",
};
private readonly HtmlParser _parser;
public HtmlContentSanitizer()
{
_parser = new HtmlParser(new HtmlParserOptions
{
IsKeepingSourceReferences = false,
});
}
/// <summary>
/// Sanitizes <paramref name="html"/> and returns a safe fragment suitable for rendering.
/// </summary>
public string Sanitize(string? html, Uri? baseUri = null)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var document = _parser.ParseDocument(html);
if (document.Body is null)
{
return string.Empty;
}
foreach (var element in document.All.ToList())
{
if (IsDangerous(element))
{
element.Remove();
continue;
}
if (!AllowedElements.Contains(element.LocalName))
{
var owner = element.Owner;
if (owner is null)
{
element.Remove();
continue;
}
var text = element.TextContent ?? string.Empty;
element.Replace(owner.CreateTextNode(text));
continue;
}
CleanAttributes(element, baseUri);
}
private static readonly HashSet<string> UrlAttributes = new(StringComparer.OrdinalIgnoreCase)
{
"href", "src",
};
private readonly HtmlParser _parser;
public HtmlContentSanitizer()
{
_parser = new HtmlParser(new HtmlParserOptions
{
IsKeepingSourceReferences = false,
});
}
/// <summary>
/// Sanitizes <paramref name="html"/> and returns a safe fragment suitable for rendering.
/// </summary>
public string Sanitize(string? html, Uri? baseUri = null)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var document = _parser.ParseDocument(html);
if (document.Body is null)
{
return string.Empty;
}
foreach (var element in document.All.ToList())
{
if (IsDangerous(element))
{
element.Remove();
continue;
}
if (!AllowedElements.Contains(element.LocalName))
{
var owner = element.Owner;
if (owner is null)
{
element.Remove();
continue;
}
var text = element.TextContent ?? string.Empty;
element.Replace(owner.CreateTextNode(text));
continue;
}
CleanAttributes(element, baseUri);
}
var body = document.Body ?? document.DocumentElement;
if (body is null)
{
@@ -82,22 +82,22 @@ public sealed class HtmlContentSanitizer
var innerHtml = body.InnerHtml;
return string.IsNullOrWhiteSpace(innerHtml) ? string.Empty : innerHtml.Trim();
}
private static bool IsDangerous(IElement element)
{
if (string.Equals(element.LocalName, "script", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "style", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "iframe", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "object", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "embed", StringComparison.OrdinalIgnoreCase))
{
return true;
}
return false;
}
}
private static bool IsDangerous(IElement element)
{
if (string.Equals(element.LocalName, "script", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "style", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "iframe", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "object", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "embed", StringComparison.OrdinalIgnoreCase))
{
return true;
}
return false;
}
private static void CleanAttributes(IElement element, Uri? baseUri)
{
if (element.Attributes is null || element.Attributes.Length == 0)
@@ -111,70 +111,70 @@ public sealed class HtmlContentSanitizer
{
element.RemoveAttribute(attribute.Name);
continue;
}
if (UrlAttributes.Contains(attribute.Name))
{
NormalizeUrlAttribute(element, attribute, baseUri);
continue;
}
if (!IsAttributeAllowed(element.LocalName, attribute.Name))
{
element.RemoveAttribute(attribute.Name);
}
}
}
private static bool IsAttributeAllowed(string elementName, string attributeName)
{
if (string.Equals(attributeName, "title", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "a", StringComparison.OrdinalIgnoreCase)
&& string.Equals(attributeName, "rel", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "table", StringComparison.OrdinalIgnoreCase)
&& (string.Equals(attributeName, "border", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellpadding", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellspacing", StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return false;
}
private static void NormalizeUrlAttribute(IElement element, IAttr attribute, Uri? baseUri)
{
if (string.IsNullOrWhiteSpace(attribute.Value))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (!UrlNormalizer.TryNormalize(attribute.Value, baseUri, out var normalized))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (string.Equals(element.LocalName, "a", StringComparison.OrdinalIgnoreCase))
{
element.SetAttribute("rel", "noopener nofollow noreferrer");
}
if (normalized is null)
{
element.RemoveAttribute(attribute.Name);
return;
}
element.SetAttribute(attribute.Name, normalized.ToString());
}
}
}
if (UrlAttributes.Contains(attribute.Name))
{
NormalizeUrlAttribute(element, attribute, baseUri);
continue;
}
if (!IsAttributeAllowed(element.LocalName, attribute.Name))
{
element.RemoveAttribute(attribute.Name);
}
}
}
private static bool IsAttributeAllowed(string elementName, string attributeName)
{
if (string.Equals(attributeName, "title", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "a", StringComparison.OrdinalIgnoreCase)
&& string.Equals(attributeName, "rel", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (string.Equals(elementName, "table", StringComparison.OrdinalIgnoreCase)
&& (string.Equals(attributeName, "border", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellpadding", StringComparison.OrdinalIgnoreCase)
|| string.Equals(attributeName, "cellspacing", StringComparison.OrdinalIgnoreCase)))
{
return true;
}
return false;
}
private static void NormalizeUrlAttribute(IElement element, IAttr attribute, Uri? baseUri)
{
if (string.IsNullOrWhiteSpace(attribute.Value))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (!UrlNormalizer.TryNormalize(attribute.Value, baseUri, out var normalized))
{
element.RemoveAttribute(attribute.Name);
return;
}
if (string.Equals(element.LocalName, "a", StringComparison.OrdinalIgnoreCase))
{
element.SetAttribute("rel", "noopener nofollow noreferrer");
}
if (normalized is null)
{
element.RemoveAttribute(attribute.Name);
return;
}
element.SetAttribute(attribute.Name, normalized.ToString());
}
}

72
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/AllowlistedHttpMessageHandler.cs
View File

@@ -1,36 +1,36 @@
using System.Net.Http.Headers;
namespace StellaOps.Concelier.Connector.Common.Http;
/// <summary>
/// Delegating handler that enforces an allowlist of destination hosts for outbound requests.
/// </summary>
internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler
{
private readonly IReadOnlyCollection<string> _allowedHosts;
public AllowlistedHttpMessageHandler(SourceHttpClientOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var snapshot = options.GetAllowedHostsSnapshot();
if (snapshot.Count == 0)
{
throw new InvalidOperationException("Source HTTP client must configure at least one allowed host.");
}
_allowedHosts = snapshot;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var host = request.RequestUri?.Host;
if (string.IsNullOrWhiteSpace(host) || !_allowedHosts.Contains(host))
{
throw new InvalidOperationException($"Request host '{host ?? "<null>"}' is not allowlisted for this source.");
}
return base.SendAsync(request, cancellationToken);
}
}
using System.Net.Http.Headers;
namespace StellaOps.Concelier.Connector.Common.Http;
/// <summary>
/// Delegating handler that enforces an allowlist of destination hosts for outbound requests.
/// </summary>
internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler
{
private readonly IReadOnlyCollection<string> _allowedHosts;
public AllowlistedHttpMessageHandler(SourceHttpClientOptions options)
{
ArgumentNullException.ThrowIfNull(options);
var snapshot = options.GetAllowedHostsSnapshot();
if (snapshot.Count == 0)
{
throw new InvalidOperationException("Source HTTP client must configure at least one allowed host.");
}
_allowedHosts = snapshot;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var host = request.RequestUri?.Host;
if (string.IsNullOrWhiteSpace(host) || !_allowedHosts.Contains(host))
{
throw new InvalidOperationException($"Request host '{host ?? "<null>"}' is not allowlisted for this source.");
}
return base.SendAsync(request, cancellationToken);
}
}

652
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientConfigurationBinder.cs
View File

@@ -1,184 +1,184 @@
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.IO;
using System.Net.Security;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Http;
internal static class SourceHttpClientConfigurationBinder
{
private const string ConcelierSection = "concelier";
private const string HttpClientsSection = "httpClients";
private const string SourcesSection = "sources";
private const string HttpSection = "http";
private const string AllowInvalidKey = "allowInvalidCertificates";
private const string TrustedRootPathsKey = "trustedRootPaths";
private const string ProxySection = "proxy";
private const string ProxyAddressKey = "address";
private const string ProxyBypassOnLocalKey = "bypassOnLocal";
private const string ProxyBypassListKey = "bypassList";
private const string ProxyUseDefaultCredentialsKey = "useDefaultCredentials";
private const string ProxyUsernameKey = "username";
private const string ProxyPasswordKey = "password";
private const string OfflineRootKey = "offlineRoot";
private const string OfflineRootEnvironmentVariable = "CONCELIER_OFFLINE_ROOT";
public static void Apply(IServiceProvider services, string clientName, SourceHttpClientOptions options)
{
var configuration = services.GetService(typeof(IConfiguration)) as IConfiguration;
if (configuration is null)
{
return;
}
var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory;
var logger = loggerFactory?.CreateLogger("SourceHttpClientConfiguration");
var hostEnvironment = services.GetService(typeof(IHostEnvironment)) as IHostEnvironment;
var processed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var section in EnumerateCandidateSections(configuration, clientName))
{
if (section is null || !section.Exists() || !processed.Add(section.Path))
{
continue;
}
ApplySection(section, configuration, hostEnvironment, clientName, options, logger);
}
}
private static IEnumerable<IConfigurationSection> EnumerateCandidateSections(IConfiguration configuration, string clientName)
{
var names = BuildCandidateNames(clientName);
foreach (var name in names)
{
var httpClientSection = GetSection(configuration, ConcelierSection, HttpClientsSection, name);
if (httpClientSection is not null && httpClientSection.Exists())
{
yield return httpClientSection;
}
var sourceHttpSection = GetSection(configuration, ConcelierSection, SourcesSection, name, HttpSection);
if (sourceHttpSection is not null && sourceHttpSection.Exists())
{
yield return sourceHttpSection;
}
}
}
private static IEnumerable<string> BuildCandidateNames(string clientName)
{
yield return clientName;
if (clientName.StartsWith("source.", StringComparison.OrdinalIgnoreCase) && clientName.Length > "source.".Length)
{
yield return clientName["source.".Length..];
}
var noDots = clientName.Replace('.', '_');
if (!string.Equals(noDots, clientName, StringComparison.OrdinalIgnoreCase))
{
yield return noDots;
}
}
private static IConfigurationSection? GetSection(IConfiguration configuration, params string[] pathSegments)
{
IConfiguration? current = configuration;
foreach (var segment in pathSegments)
{
if (current is null)
{
return null;
}
current = current.GetSection(segment);
}
return current as IConfigurationSection;
}
private static void ApplySection(
IConfigurationSection section,
IConfiguration rootConfiguration,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var allowInvalid = section.GetValue<bool?>(AllowInvalidKey);
if (allowInvalid == true)
{
options.AllowInvalidServerCertificates = true;
var previous = options.ServerCertificateCustomValidation;
options.ServerCertificateCustomValidation = (certificate, chain, errors) =>
{
if (allowInvalid == true)
{
return true;
}
return previous?.Invoke(certificate, chain, errors) ?? errors == SslPolicyErrors.None;
};
logger?.LogWarning(
"Source HTTP client '{ClientName}' is configured to bypass TLS certificate validation.",
clientName);
}
var offlineRoot = section.GetValue<string?>(OfflineRootKey)
?? rootConfiguration.GetSection(ConcelierSection).GetValue<string?>(OfflineRootKey)
?? Environment.GetEnvironmentVariable(OfflineRootEnvironmentVariable);
ApplyTrustedRoots(section, offlineRoot, hostEnvironment, clientName, options, logger);
ApplyProxyConfiguration(section, clientName, options, logger);
}
private static void ApplyTrustedRoots(
IConfigurationSection section,
string? offlineRoot,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var trustedRootSection = section.GetSection(TrustedRootPathsKey);
if (!trustedRootSection.Exists())
{
return;
}
var paths = trustedRootSection.Get<string[]?>();
if (paths is null || paths.Length == 0)
{
return;
}
foreach (var rawPath in paths)
{
if (string.IsNullOrWhiteSpace(rawPath))
{
continue;
}
var resolvedPath = ResolvePath(rawPath, offlineRoot, hostEnvironment);
if (!File.Exists(resolvedPath))
{
var message = string.Format(
CultureInfo.InvariantCulture,
"Trusted root certificate '{0}' resolved to '{1}' but was not found.",
rawPath,
resolvedPath);
throw new FileNotFoundException(message, resolvedPath);
}
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.IO;
using System.Net.Security;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Http;
internal static class SourceHttpClientConfigurationBinder
{
private const string ConcelierSection = "concelier";
private const string HttpClientsSection = "httpClients";
private const string SourcesSection = "sources";
private const string HttpSection = "http";
private const string AllowInvalidKey = "allowInvalidCertificates";
private const string TrustedRootPathsKey = "trustedRootPaths";
private const string ProxySection = "proxy";
private const string ProxyAddressKey = "address";
private const string ProxyBypassOnLocalKey = "bypassOnLocal";
private const string ProxyBypassListKey = "bypassList";
private const string ProxyUseDefaultCredentialsKey = "useDefaultCredentials";
private const string ProxyUsernameKey = "username";
private const string ProxyPasswordKey = "password";
private const string OfflineRootKey = "offlineRoot";
private const string OfflineRootEnvironmentVariable = "CONCELIER_OFFLINE_ROOT";
public static void Apply(IServiceProvider services, string clientName, SourceHttpClientOptions options)
{
var configuration = services.GetService(typeof(IConfiguration)) as IConfiguration;
if (configuration is null)
{
return;
}
var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory;
var logger = loggerFactory?.CreateLogger("SourceHttpClientConfiguration");
var hostEnvironment = services.GetService(typeof(IHostEnvironment)) as IHostEnvironment;
var processed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var section in EnumerateCandidateSections(configuration, clientName))
{
if (section is null || !section.Exists() || !processed.Add(section.Path))
{
continue;
}
ApplySection(section, configuration, hostEnvironment, clientName, options, logger);
}
}
private static IEnumerable<IConfigurationSection> EnumerateCandidateSections(IConfiguration configuration, string clientName)
{
var names = BuildCandidateNames(clientName);
foreach (var name in names)
{
var httpClientSection = GetSection(configuration, ConcelierSection, HttpClientsSection, name);
if (httpClientSection is not null && httpClientSection.Exists())
{
yield return httpClientSection;
}
var sourceHttpSection = GetSection(configuration, ConcelierSection, SourcesSection, name, HttpSection);
if (sourceHttpSection is not null && sourceHttpSection.Exists())
{
yield return sourceHttpSection;
}
}
}
private static IEnumerable<string> BuildCandidateNames(string clientName)
{
yield return clientName;
if (clientName.StartsWith("source.", StringComparison.OrdinalIgnoreCase) && clientName.Length > "source.".Length)
{
yield return clientName["source.".Length..];
}
var noDots = clientName.Replace('.', '_');
if (!string.Equals(noDots, clientName, StringComparison.OrdinalIgnoreCase))
{
yield return noDots;
}
}
private static IConfigurationSection? GetSection(IConfiguration configuration, params string[] pathSegments)
{
IConfiguration? current = configuration;
foreach (var segment in pathSegments)
{
if (current is null)
{
return null;
}
current = current.GetSection(segment);
}
return current as IConfigurationSection;
}
private static void ApplySection(
IConfigurationSection section,
IConfiguration rootConfiguration,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var allowInvalid = section.GetValue<bool?>(AllowInvalidKey);
if (allowInvalid == true)
{
options.AllowInvalidServerCertificates = true;
var previous = options.ServerCertificateCustomValidation;
options.ServerCertificateCustomValidation = (certificate, chain, errors) =>
{
if (allowInvalid == true)
{
return true;
}
return previous?.Invoke(certificate, chain, errors) ?? errors == SslPolicyErrors.None;
};
logger?.LogWarning(
"Source HTTP client '{ClientName}' is configured to bypass TLS certificate validation.",
clientName);
}
var offlineRoot = section.GetValue<string?>(OfflineRootKey)
?? rootConfiguration.GetSection(ConcelierSection).GetValue<string?>(OfflineRootKey)
?? Environment.GetEnvironmentVariable(OfflineRootEnvironmentVariable);
ApplyTrustedRoots(section, offlineRoot, hostEnvironment, clientName, options, logger);
ApplyProxyConfiguration(section, clientName, options, logger);
}
private static void ApplyTrustedRoots(
IConfigurationSection section,
string? offlineRoot,
IHostEnvironment? hostEnvironment,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var trustedRootSection = section.GetSection(TrustedRootPathsKey);
if (!trustedRootSection.Exists())
{
return;
}
var paths = trustedRootSection.Get<string[]?>();
if (paths is null || paths.Length == 0)
{
return;
}
foreach (var rawPath in paths)
{
if (string.IsNullOrWhiteSpace(rawPath))
{
continue;
}
var resolvedPath = ResolvePath(rawPath, offlineRoot, hostEnvironment);
if (!File.Exists(resolvedPath))
{
var message = string.Format(
CultureInfo.InvariantCulture,
"Trusted root certificate '{0}' resolved to '{1}' but was not found.",
rawPath,
resolvedPath);
throw new FileNotFoundException(message, resolvedPath);
}
foreach (var certificate in LoadCertificates(resolvedPath))
{
var thumbprint = certificate.Thumbprint;
@@ -194,134 +194,134 @@ internal static class SourceHttpClientConfigurationBinder
}
}
}
private static void ApplyProxyConfiguration(
IConfigurationSection section,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var proxySection = section.GetSection(ProxySection);
if (!proxySection.Exists())
{
return;
}
var address = proxySection.GetValue<string?>(ProxyAddressKey);
if (!string.IsNullOrWhiteSpace(address))
{
if (Uri.TryCreate(address, UriKind.Absolute, out var uri))
{
options.ProxyAddress = uri;
}
else
{
logger?.LogWarning(
"Source HTTP client '{ClientName}' has invalid proxy address '{ProxyAddress}'.",
clientName,
address);
}
}
var bypassOnLocal = proxySection.GetValue<bool?>(ProxyBypassOnLocalKey);
if (bypassOnLocal.HasValue)
{
options.ProxyBypassOnLocal = bypassOnLocal.Value;
}
var bypassListSection = proxySection.GetSection(ProxyBypassListKey);
if (bypassListSection.Exists())
{
var entries = bypassListSection.Get<string[]?>();
options.ProxyBypassList.Clear();
if (entries is not null)
{
foreach (var entry in entries)
{
if (!string.IsNullOrWhiteSpace(entry))
{
options.ProxyBypassList.Add(entry.Trim());
}
}
}
}
var useDefaultCredentials = proxySection.GetValue<bool?>(ProxyUseDefaultCredentialsKey);
if (useDefaultCredentials.HasValue)
{
options.ProxyUseDefaultCredentials = useDefaultCredentials.Value;
}
var username = proxySection.GetValue<string?>(ProxyUsernameKey);
if (!string.IsNullOrWhiteSpace(username))
{
options.ProxyUsername = username.Trim();
}
var password = proxySection.GetValue<string?>(ProxyPasswordKey);
if (!string.IsNullOrWhiteSpace(password))
{
options.ProxyPassword = password;
}
}
private static string ResolvePath(string path, string? offlineRoot, IHostEnvironment? hostEnvironment)
{
if (Path.IsPathRooted(path))
{
return path;
}
if (!string.IsNullOrWhiteSpace(offlineRoot))
{
return Path.GetFullPath(Path.Combine(offlineRoot!, path));
}
var baseDirectory = hostEnvironment?.ContentRootPath ?? AppContext.BaseDirectory;
return Path.GetFullPath(Path.Combine(baseDirectory, path));
}
private static IEnumerable<X509Certificate2> LoadCertificates(string path)
{
var certificates = new List<X509Certificate2>();
var extension = Path.GetExtension(path);
if (extension.Equals(".pem", StringComparison.OrdinalIgnoreCase) || extension.Equals(".crt", StringComparison.OrdinalIgnoreCase))
{
var collection = new X509Certificate2Collection();
try
{
collection.ImportFromPemFile(path);
}
catch (CryptographicException)
{
collection.Clear();
}
if (collection.Count > 0)
{
foreach (var certificate in collection)
{
certificates.Add(certificate.CopyWithPrivateKeyIfAvailable());
}
}
else
{
certificates.Add(X509Certificate2.CreateFromPemFile(path));
}
}
else
{
// Use X509CertificateLoader to load certificates from PKCS#12 files (.pfx, .p12, etc.)
var certificate = System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12(
File.ReadAllBytes(path),
password: null);
certificates.Add(certificate);
}
return certificates;
}
private static void ApplyProxyConfiguration(
IConfigurationSection section,
string clientName,
SourceHttpClientOptions options,
ILogger? logger)
{
var proxySection = section.GetSection(ProxySection);
if (!proxySection.Exists())
{
return;
}
var address = proxySection.GetValue<string?>(ProxyAddressKey);
if (!string.IsNullOrWhiteSpace(address))
{
if (Uri.TryCreate(address, UriKind.Absolute, out var uri))
{
options.ProxyAddress = uri;
}
else
{
logger?.LogWarning(
"Source HTTP client '{ClientName}' has invalid proxy address '{ProxyAddress}'.",
clientName,
address);
}
}
var bypassOnLocal = proxySection.GetValue<bool?>(ProxyBypassOnLocalKey);
if (bypassOnLocal.HasValue)
{
options.ProxyBypassOnLocal = bypassOnLocal.Value;
}
var bypassListSection = proxySection.GetSection(ProxyBypassListKey);
if (bypassListSection.Exists())
{
var entries = bypassListSection.Get<string[]?>();
options.ProxyBypassList.Clear();
if (entries is not null)
{
foreach (var entry in entries)
{
if (!string.IsNullOrWhiteSpace(entry))
{
options.ProxyBypassList.Add(entry.Trim());
}
}
}
}
var useDefaultCredentials = proxySection.GetValue<bool?>(ProxyUseDefaultCredentialsKey);
if (useDefaultCredentials.HasValue)
{
options.ProxyUseDefaultCredentials = useDefaultCredentials.Value;
}
var username = proxySection.GetValue<string?>(ProxyUsernameKey);
if (!string.IsNullOrWhiteSpace(username))
{
options.ProxyUsername = username.Trim();
}
var password = proxySection.GetValue<string?>(ProxyPasswordKey);
if (!string.IsNullOrWhiteSpace(password))
{
options.ProxyPassword = password;
}
}
private static string ResolvePath(string path, string? offlineRoot, IHostEnvironment? hostEnvironment)
{
if (Path.IsPathRooted(path))
{
return path;
}
if (!string.IsNullOrWhiteSpace(offlineRoot))
{
return Path.GetFullPath(Path.Combine(offlineRoot!, path));
}
var baseDirectory = hostEnvironment?.ContentRootPath ?? AppContext.BaseDirectory;
return Path.GetFullPath(Path.Combine(baseDirectory, path));
}
private static IEnumerable<X509Certificate2> LoadCertificates(string path)
{
var certificates = new List<X509Certificate2>();
var extension = Path.GetExtension(path);
if (extension.Equals(".pem", StringComparison.OrdinalIgnoreCase) || extension.Equals(".crt", StringComparison.OrdinalIgnoreCase))
{
var collection = new X509Certificate2Collection();
try
{
collection.ImportFromPemFile(path);
}
catch (CryptographicException)
{
collection.Clear();
}
if (collection.Count > 0)
{
foreach (var certificate in collection)
{
certificates.Add(certificate.CopyWithPrivateKeyIfAvailable());
}
}
else
{
certificates.Add(X509Certificate2.CreateFromPemFile(path));
}
}
else
{
// Use X509CertificateLoader to load certificates from PKCS#12 files (.pfx, .p12, etc.)
var certificate = System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12(
File.ReadAllBytes(path),
password: null);
certificates.Add(certificate);
}
return certificates;
}
private static bool AddTrustedCertificate(SourceHttpClientOptions options, X509Certificate2 certificate)
{
if (certificate is null)
@@ -347,20 +347,20 @@ internal static class SourceHttpClientConfigurationBinder
return true;
}
// Helper extension method to copy certificate (preserves private key if present)
private static X509Certificate2 CopyWithPrivateKeyIfAvailable(this X509Certificate2 certificate)
{
// In .NET 9+, use X509CertificateLoader instead of obsolete constructors
if (certificate.HasPrivateKey)
{
// Export with private key and re-import using X509CertificateLoader
var exported = certificate.Export(X509ContentType.Pkcs12);
return X509CertificateLoader.LoadPkcs12(exported, password: null);
}
else
{
// For certificates without private keys, load from raw data
return X509CertificateLoader.LoadCertificate(certificate.RawData);
}
}
}
// Helper extension method to copy certificate (preserves private key if present)
private static X509Certificate2 CopyWithPrivateKeyIfAvailable(this X509Certificate2 certificate)
{
// In .NET 9+, use X509CertificateLoader instead of obsolete constructors
if (certificate.HasPrivateKey)
{
// Export with private key and re-import using X509CertificateLoader
var exported = certificate.Export(X509ContentType.Pkcs12);
return X509CertificateLoader.LoadPkcs12(exported, password: null);
}
else
{
// For certificates without private keys, load from raw data
return X509CertificateLoader.LoadCertificate(certificate.RawData);
}
}
}

78
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientOptions.cs
View File

@@ -8,43 +8,43 @@ namespace StellaOps.Concelier.Connector.Common.Http;
/// <summary>
/// Configuration applied to named HTTP clients used by connectors.
/// </summary>
public sealed class SourceHttpClientOptions
{
private readonly HashSet<string> _allowedHosts = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, string> _defaultHeaders = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Gets or sets the base address used for relative requests.
/// </summary>
public Uri? BaseAddress { get; set; }
/// <summary>
/// Gets or sets the client timeout.
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Gets or sets the user-agent string applied to outgoing requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps.Concelier/1.0";
/// <summary>
/// Gets or sets whether redirects are allowed. Defaults to <c>true</c>.
/// </summary>
public bool AllowAutoRedirect { get; set; } = true;
/// <summary>
/// </summary>
public sealed class SourceHttpClientOptions
{
private readonly HashSet<string> _allowedHosts = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, string> _defaultHeaders = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Gets or sets the base address used for relative requests.
/// </summary>
public Uri? BaseAddress { get; set; }
/// <summary>
/// Gets or sets the client timeout.
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Gets or sets the user-agent string applied to outgoing requests.
/// </summary>
public string UserAgent { get; set; } = "StellaOps.Concelier/1.0";
/// <summary>
/// Gets or sets whether redirects are allowed. Defaults to <c>true</c>.
/// </summary>
public bool AllowAutoRedirect { get; set; } = true;
/// <summary>
/// Maximum number of retry attempts for transient failures.
/// </summary>
public int MaxAttempts { get; set; } = 3;
/// <summary>
/// Base delay applied to the exponential backoff policy.
/// </summary>
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Base delay applied to the exponential backoff policy.
/// </summary>
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Hosts that this client is allowed to contact.
/// </summary>
public ISet<string> AllowedHosts => _allowedHosts;
@@ -120,10 +120,10 @@ public sealed class SourceHttpClientOptions
public IDictionary<string, string> DefaultRequestHeaders => _defaultHeaders;
internal SourceHttpClientOptions Clone()
{
var clone = new SourceHttpClientOptions
{
BaseAddress = BaseAddress,
{
var clone = new SourceHttpClientOptions
{
BaseAddress = BaseAddress,
Timeout = Timeout,
UserAgent = UserAgent,
AllowAutoRedirect = AllowAutoRedirect,
@@ -145,8 +145,8 @@ public sealed class SourceHttpClientOptions
foreach (var host in _allowedHosts)
{
clone.AllowedHosts.Add(host);
}
}
foreach (var header in _defaultHeaders)
{
clone.DefaultRequestHeaders[header.Key] = header.Value;
@@ -167,4 +167,4 @@ public sealed class SourceHttpClientOptions
internal IReadOnlyCollection<string> GetAllowedHostsSnapshot()
=> new ReadOnlyCollection<string>(_allowedHosts.ToArray());
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/IJsonSchemaValidator.cs
View File

@@ -1,9 +1,9 @@
using System.Text.Json;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Common.Json;
public interface IJsonSchemaValidator
{
void Validate(JsonDocument document, JsonSchema schema, string documentName);
}
using System.Text.Json;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Common.Json;
public interface IJsonSchemaValidator
{
void Validate(JsonDocument document, JsonSchema schema, string documentName);
}

14
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationError.cs
View File

@@ -1,7 +1,7 @@
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed record JsonSchemaValidationError(
string InstanceLocation,
string SchemaLocation,
string Message,
string Keyword);
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed record JsonSchemaValidationError(
string InstanceLocation,
string SchemaLocation,
string Message,
string Keyword);

30
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationException.cs
View File

@@ -1,15 +1,15 @@
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidationException : Exception
{
public JsonSchemaValidationException(string documentName, IReadOnlyList<JsonSchemaValidationError> errors)
: base($"JSON schema validation failed for '{documentName}'.")
{
DocumentName = documentName;
Errors = errors ?? Array.Empty<JsonSchemaValidationError>();
}
public string DocumentName { get; }
public IReadOnlyList<JsonSchemaValidationError> Errors { get; }
}
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidationException : Exception
{
public JsonSchemaValidationException(string documentName, IReadOnlyList<JsonSchemaValidationError> errors)
: base($"JSON schema validation failed for '{documentName}'.")
{
DocumentName = documentName;
Errors = errors ?? Array.Empty<JsonSchemaValidationError>();
}
public string DocumentName { get; }
public IReadOnlyList<JsonSchemaValidationError> Errors { get; }
}

184
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidator.cs
View File

@@ -1,92 +1,92 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Json.Schema;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidator : IJsonSchemaValidator
{
private readonly ILogger<JsonSchemaValidator> _logger;
private const int MaxLoggedErrors = 5;
public JsonSchemaValidator(ILogger<JsonSchemaValidator> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public void Validate(JsonDocument document, JsonSchema schema, string documentName)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(schema);
ArgumentException.ThrowIfNullOrEmpty(documentName);
var result = schema.Evaluate(document.RootElement, new EvaluationOptions
{
OutputFormat = OutputFormat.List,
RequireFormatValidation = true,
});
if (result.IsValid)
{
return;
}
var errors = CollectErrors(result);
if (errors.Count == 0)
{
_logger.LogWarning("Schema validation failed for {Document} with unknown errors", documentName);
throw new JsonSchemaValidationException(documentName, errors);
}
foreach (var violation in errors.Take(MaxLoggedErrors))
{
_logger.LogWarning(
"Schema violation for {Document} at {InstanceLocation} (keyword: {Keyword}): {Message}",
documentName,
string.IsNullOrEmpty(violation.InstanceLocation) ? "#" : violation.InstanceLocation,
violation.Keyword,
violation.Message);
}
if (errors.Count > MaxLoggedErrors)
{
_logger.LogWarning("{Count} additional schema violations for {Document} suppressed", errors.Count - MaxLoggedErrors, documentName);
}
throw new JsonSchemaValidationException(documentName, errors);
}
private static IReadOnlyList<JsonSchemaValidationError> CollectErrors(EvaluationResults result)
{
var errors = new List<JsonSchemaValidationError>();
Aggregate(result, errors);
return errors;
}
private static void Aggregate(EvaluationResults node, List<JsonSchemaValidationError> errors)
{
if (node.Errors is { Count: > 0 })
{
foreach (var kvp in node.Errors)
{
errors.Add(new JsonSchemaValidationError(
node.InstanceLocation?.ToString() ?? string.Empty,
node.SchemaLocation?.ToString() ?? string.Empty,
kvp.Value,
kvp.Key));
}
}
if (node.Details is null)
{
return;
}
foreach (var child in node.Details)
{
Aggregate(child, errors);
}
}
}
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Json.Schema;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Json;
public sealed class JsonSchemaValidator : IJsonSchemaValidator
{
private readonly ILogger<JsonSchemaValidator> _logger;
private const int MaxLoggedErrors = 5;
public JsonSchemaValidator(ILogger<JsonSchemaValidator> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public void Validate(JsonDocument document, JsonSchema schema, string documentName)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(schema);
ArgumentException.ThrowIfNullOrEmpty(documentName);
var result = schema.Evaluate(document.RootElement, new EvaluationOptions
{
OutputFormat = OutputFormat.List,
RequireFormatValidation = true,
});
if (result.IsValid)
{
return;
}
var errors = CollectErrors(result);
if (errors.Count == 0)
{
_logger.LogWarning("Schema validation failed for {Document} with unknown errors", documentName);
throw new JsonSchemaValidationException(documentName, errors);
}
foreach (var violation in errors.Take(MaxLoggedErrors))
{
_logger.LogWarning(
"Schema violation for {Document} at {InstanceLocation} (keyword: {Keyword}): {Message}",
documentName,
string.IsNullOrEmpty(violation.InstanceLocation) ? "#" : violation.InstanceLocation,
violation.Keyword,
violation.Message);
}
if (errors.Count > MaxLoggedErrors)
{
_logger.LogWarning("{Count} additional schema violations for {Document} suppressed", errors.Count - MaxLoggedErrors, documentName);
}
throw new JsonSchemaValidationException(documentName, errors);
}
private static IReadOnlyList<JsonSchemaValidationError> CollectErrors(EvaluationResults result)
{
var errors = new List<JsonSchemaValidationError>();
Aggregate(result, errors);
return errors;
}
private static void Aggregate(EvaluationResults node, List<JsonSchemaValidationError> errors)
{
if (node.Errors is { Count: > 0 })
{
foreach (var kvp in node.Errors)
{
errors.Add(new JsonSchemaValidationError(
node.InstanceLocation?.ToString() ?? string.Empty,
node.SchemaLocation?.ToString() ?? string.Empty,
kvp.Value,
kvp.Key));
}
}
if (node.Details is null)
{
return;
}
foreach (var child in node.Details)
{
Aggregate(child, errors);
}
}
}

242
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Packages/PackageCoordinateHelper.cs
View File

@@ -1,23 +1,23 @@
using System.Linq;
using System.Text;
using NuGet.Versioning;
using StellaOps.Concelier.Normalization.Identifiers;
namespace StellaOps.Concelier.Connector.Common.Packages;
/// <summary>
/// Shared helpers for working with Package URLs and SemVer coordinates inside connectors.
/// </summary>
public static class PackageCoordinateHelper
{
public static bool TryParsePackageUrl(string? value, out PackageCoordinates? coordinates)
{
coordinates = null;
if (!IdentifierNormalizer.TryNormalizePackageUrl(value, out var canonical, out var packageUrl) || packageUrl is null)
{
return false;
}
using System.Linq;
using System.Text;
using NuGet.Versioning;
using StellaOps.Concelier.Normalization.Identifiers;
namespace StellaOps.Concelier.Connector.Common.Packages;
/// <summary>
/// Shared helpers for working with Package URLs and SemVer coordinates inside connectors.
/// </summary>
public static class PackageCoordinateHelper
{
public static bool TryParsePackageUrl(string? value, out PackageCoordinates? coordinates)
{
coordinates = null;
if (!IdentifierNormalizer.TryNormalizePackageUrl(value, out var canonical, out var packageUrl) || packageUrl is null)
{
return false;
}
var namespaceSegments = packageUrl.NamespaceSegments.ToArray();
var subpathSegments = packageUrl.SubpathSegments.ToArray();
var qualifiers = packageUrl.Qualifiers.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase);
@@ -39,46 +39,46 @@ public static class PackageCoordinateHelper
SubpathSegments: subpathSegments,
Original: packageUrl.Original);
return true;
}
public static PackageCoordinates ParsePackageUrl(string value)
{
if (!TryParsePackageUrl(value, out var coordinates) || coordinates is null)
{
throw new FormatException($"Value '{value}' is not a valid Package URL");
}
return coordinates;
}
public static bool TryParseSemVer(string? value, out SemanticVersion? version, out string? normalized)
{
version = null;
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!SemanticVersion.TryParse(value.Trim(), out var parsed))
{
return false;
}
version = parsed;
normalized = parsed.ToNormalizedString();
return true;
}
public static bool TryParseSemVerRange(string? value, out VersionRange? range)
{
range = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
}
public static PackageCoordinates ParsePackageUrl(string value)
{
if (!TryParsePackageUrl(value, out var coordinates) || coordinates is null)
{
throw new FormatException($"Value '{value}' is not a valid Package URL");
}
return coordinates;
}
public static bool TryParseSemVer(string? value, out SemanticVersion? version, out string? normalized)
{
version = null;
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!SemanticVersion.TryParse(value.Trim(), out var parsed))
{
return false;
}
version = parsed;
normalized = parsed.ToNormalizedString();
return true;
}
public static bool TryParseSemVerRange(string? value, out VersionRange? range)
{
range = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
var trimmed = value.Trim();
if (trimmed.StartsWith("^", StringComparison.Ordinal))
{
@@ -113,57 +113,57 @@ public static class PackageCoordinateHelper
range = parsed;
return true;
}
public static string BuildPackageUrl(
string type,
IReadOnlyList<string>? namespaceSegments,
string name,
string? version = null,
IReadOnlyDictionary<string, string>? qualifiers = null,
IReadOnlyList<string>? subpathSegments = null)
{
ArgumentException.ThrowIfNullOrEmpty(type);
ArgumentException.ThrowIfNullOrEmpty(name);
var builder = new StringBuilder("pkg:");
builder.Append(type.Trim().ToLowerInvariant());
builder.Append('/');
if (namespaceSegments is not null && namespaceSegments.Count > 0)
{
builder.Append(string.Join('/', namespaceSegments.Select(NormalizeSegment)));
builder.Append('/');
}
builder.Append(NormalizeSegment(name));
if (!string.IsNullOrWhiteSpace(version))
{
builder.Append('@');
builder.Append(version.Trim());
}
if (qualifiers is not null && qualifiers.Count > 0)
{
builder.Append('?');
builder.Append(string.Join('&', qualifiers
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
.Select(kvp => $"{NormalizeSegment(kvp.Key)}={NormalizeSegment(kvp.Value)}")));
}
if (subpathSegments is not null && subpathSegments.Count > 0)
{
builder.Append('#');
builder.Append(string.Join('/', subpathSegments.Select(NormalizeSegment)));
}
return builder.ToString();
}
private static string NormalizeSegment(string value)
{
ArgumentNullException.ThrowIfNull(value);
}
public static string BuildPackageUrl(
string type,
IReadOnlyList<string>? namespaceSegments,
string name,
string? version = null,
IReadOnlyDictionary<string, string>? qualifiers = null,
IReadOnlyList<string>? subpathSegments = null)
{
ArgumentException.ThrowIfNullOrEmpty(type);
ArgumentException.ThrowIfNullOrEmpty(name);
var builder = new StringBuilder("pkg:");
builder.Append(type.Trim().ToLowerInvariant());
builder.Append('/');
if (namespaceSegments is not null && namespaceSegments.Count > 0)
{
builder.Append(string.Join('/', namespaceSegments.Select(NormalizeSegment)));
builder.Append('/');
}
builder.Append(NormalizeSegment(name));
if (!string.IsNullOrWhiteSpace(version))
{
builder.Append('@');
builder.Append(version.Trim());
}
if (qualifiers is not null && qualifiers.Count > 0)
{
builder.Append('?');
builder.Append(string.Join('&', qualifiers
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
.Select(kvp => $"{NormalizeSegment(kvp.Key)}={NormalizeSegment(kvp.Value)}")));
}
if (subpathSegments is not null && subpathSegments.Count > 0)
{
builder.Append('#');
builder.Append(string.Join('/', subpathSegments.Select(NormalizeSegment)));
}
return builder.ToString();
}
private static string NormalizeSegment(string value)
{
ArgumentNullException.ThrowIfNull(value);
var trimmed = value.Trim();
var unescaped = Uri.UnescapeDataString(trimmed);
var encoded = Uri.EscapeDataString(unescaped);
@@ -185,13 +185,13 @@ public static class PackageCoordinateHelper
return new SemanticVersion(0, 0, baseVersion.Patch + 1);
}
}
public sealed record PackageCoordinates(
string Canonical,
string Type,
IReadOnlyList<string> NamespaceSegments,
string Name,
string? Version,
IReadOnlyDictionary<string, string> Qualifiers,
IReadOnlyList<string> SubpathSegments,
string Original);
public sealed record PackageCoordinates(
string Canonical,
string Type,
IReadOnlyList<string> NamespaceSegments,
string Name,
string? Version,
IReadOnlyDictionary<string, string> Qualifiers,
IReadOnlyList<string> SubpathSegments,
string Original);

144
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Pdf/PdfTextExtractor.cs
View File

@@ -2,22 +2,22 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using System.Text;
using UglyToad.PdfPig;
using UglyToad.PdfPig.Content;
namespace StellaOps.Concelier.Connector.Common.Pdf;
/// <summary>
/// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies.
/// </summary>
public sealed class PdfTextExtractor
{
public async Task<PdfExtractionResult> ExtractTextAsync(Stream pdfStream, PdfExtractionOptions? options = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(pdfStream);
options ??= PdfExtractionOptions.Default;
using System.Text;
using UglyToad.PdfPig;
using UglyToad.PdfPig.Content;
namespace StellaOps.Concelier.Connector.Common.Pdf;
/// <summary>
/// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies.
/// </summary>
public sealed class PdfTextExtractor
{
public async Task<PdfExtractionResult> ExtractTextAsync(Stream pdfStream, PdfExtractionOptions? options = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(pdfStream);
options ??= PdfExtractionOptions.Default;
using var buffer = new MemoryStream();
await pdfStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
var rawBytes = buffer.ToArray();
@@ -28,10 +28,10 @@ public sealed class PdfTextExtractor
ClipPaths = true,
UseLenientParsing = true,
});
var builder = new StringBuilder();
var pageCount = 0;
var builder = new StringBuilder();
var pageCount = 0;
var totalPages = document.NumberOfPages;
for (var index = 1; index <= totalPages; index++)
{
@@ -52,12 +52,12 @@ public sealed class PdfTextExtractor
{
break;
}
if (pageCount > 1 && options.PageSeparator is not null)
{
builder.Append(options.PageSeparator);
}
if (pageCount > 1 && options.PageSeparator is not null)
{
builder.Append(options.PageSeparator);
}
string text;
try
{
@@ -93,8 +93,8 @@ public sealed class PdfTextExtractor
{
builder.AppendLine(text.Trim());
}
}
}
if (builder.Length == 0)
{
var raw = Encoding.ASCII.GetString(rawBytes);
@@ -119,28 +119,28 @@ public sealed class PdfTextExtractor
}
return new PdfExtractionResult(builder.ToString().Trim(), pageCount);
}
private static string FlattenWords(IEnumerable<Word> words)
{
var builder = new StringBuilder();
var first = true;
foreach (var word in words)
{
if (string.IsNullOrWhiteSpace(word.Text))
{
continue;
}
if (!first)
{
builder.Append(' ');
}
builder.Append(word.Text.Trim());
first = false;
}
}
private static string FlattenWords(IEnumerable<Word> words)
{
var builder = new StringBuilder();
var first = true;
foreach (var word in words)
{
if (string.IsNullOrWhiteSpace(word.Text))
{
continue;
}
if (!first)
{
builder.Append(' ');
}
builder.Append(word.Text.Trim());
first = false;
}
return builder.ToString();
}
@@ -160,25 +160,25 @@ public sealed class PdfTextExtractor
return builder.ToString();
}
}
public sealed record PdfExtractionResult(string Text, int PagesProcessed);
public sealed record PdfExtractionOptions
{
public static PdfExtractionOptions Default { get; } = new();
/// <summary>
/// Maximum number of pages to read. Null reads the entire document.
/// </summary>
public int? MaxPages { get; init; }
/// <summary>
/// When true, uses PdfPig's native layout text. When false, collapses to a single line per page.
/// </summary>
public bool PreserveLayout { get; init; } = true;
/// <summary>
/// Separator inserted between pages. Null disables separators.
/// </summary>
public string? PageSeparator { get; init; } = "\n\n";
}
public sealed record PdfExtractionResult(string Text, int PagesProcessed);
public sealed record PdfExtractionOptions
{
public static PdfExtractionOptions Default { get; } = new();
/// <summary>
/// Maximum number of pages to read. Null reads the entire document.
/// </summary>
public int? MaxPages { get; init; }
/// <summary>
/// When true, uses PdfPig's native layout text. When false, collapses to a single line per page.
/// </summary>
public bool PreserveLayout { get; init; } = true;
/// <summary>
/// Separator inserted between pages. Null disables separators.
/// </summary>
public string? PageSeparator { get; init; } = "\n\n";
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Common.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Common.Tests")]

318
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedModels.cs
View File

@@ -1,159 +1,159 @@
using StellaOps.Concelier.Connector.Common;
namespace StellaOps.Concelier.Connector.Common.State;
/// <summary>
/// Describes a raw upstream document that should be persisted for a connector during seeding.
/// </summary>
public sealed record SourceStateSeedDocument
{
/// <summary>
/// Absolute source URI. Must match the connector's upstream document identifier.
/// </summary>
public string Uri { get; init; } = string.Empty;
/// <summary>
/// Raw document payload. Required when creating or replacing a document.
/// </summary>
public byte[] Content { get; init; } = Array.Empty<byte>();
/// <summary>
/// Optional explicit document identifier. When provided it overrides auto-generated IDs.
/// </summary>
public Guid? DocumentId { get; init; }
/// <summary>
/// MIME type for the document payload.
/// </summary>
public string? ContentType { get; init; }
/// <summary>
/// Status assigned to the document. Defaults to <see cref="DocumentStatuses.PendingParse"/>.
/// </summary>
public string Status { get; init; } = DocumentStatuses.PendingParse;
/// <summary>
/// Optional HTTP-style headers persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Headers { get; init; }
/// <summary>
/// Source metadata (connector specific) persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
/// <summary>
/// Upstream ETag value, if available.
/// </summary>
public string? Etag { get; init; }
/// <summary>
/// Upstream last-modified timestamp, if available.
/// </summary>
public DateTimeOffset? LastModified { get; init; }
/// <summary>
/// Optional document expiration. When set a TTL will purge the raw payload after the configured retention.
/// </summary>
public DateTimeOffset? ExpiresAt { get; init; }
/// <summary>
/// Fetch timestamp stamped onto the document. Defaults to the seed completion timestamp.
/// </summary>
public DateTimeOffset? FetchedAt { get; init; }
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingDocuments</c> set.
/// </summary>
public bool AddToPendingDocuments { get; init; } = true;
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingMappings</c> set.
/// </summary>
public bool AddToPendingMappings { get; init; }
/// <summary>
/// Optional identifiers that should be recorded on the cursor to avoid duplicate ingestion.
/// </summary>
public IReadOnlyCollection<string>? KnownIdentifiers { get; init; }
}
/// <summary>
/// Cursor updates that should accompany seeded documents.
/// </summary>
public sealed record SourceStateSeedCursor
{
/// <summary>
/// Optional <c>pendingDocuments</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingDocuments { get; init; }
/// <summary>
/// Optional <c>pendingMappings</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingMappings { get; init; }
/// <summary>
/// Optional known advisory identifiers to merge with the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Upstream window watermark tracked by connectors that rely on last-modified cursors.
/// </summary>
public DateTimeOffset? LastModifiedCursor { get; init; }
/// <summary>
/// Optional fetch timestamp used by connectors that track the last polling instant.
/// </summary>
public DateTimeOffset? LastFetchAt { get; init; }
/// <summary>
/// Additional cursor fields (string values) to merge.
/// </summary>
public IReadOnlyDictionary<string, string>? Additional { get; init; }
}
/// <summary>
/// Seeding specification describing the source, documents, and cursor edits to apply.
/// </summary>
public sealed record SourceStateSeedSpecification
{
/// <summary>
/// Source/connector name (e.g. <c>vndr.msrc</c>).
/// </summary>
public string Source { get; init; } = string.Empty;
/// <summary>
/// Documents that should be inserted or replaced before the cursor update.
/// </summary>
public IReadOnlyList<SourceStateSeedDocument> Documents { get; init; } = Array.Empty<SourceStateSeedDocument>();
/// <summary>
/// Cursor adjustments applied after documents are persisted.
/// </summary>
public SourceStateSeedCursor? Cursor { get; init; }
/// <summary>
/// Connector-level known advisory identifiers to merge into the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Optional completion timestamp. Defaults to the processor's time provider.
/// </summary>
public DateTimeOffset? CompletedAt { get; init; }
}
/// <summary>
/// Result returned after seeding completes.
/// </summary>
public sealed record SourceStateSeedResult(
int DocumentsProcessed,
int PendingDocumentsAdded,
int PendingMappingsAdded,
IReadOnlyCollection<Guid> DocumentIds,
IReadOnlyCollection<Guid> PendingDocumentIds,
IReadOnlyCollection<Guid> PendingMappingIds,
IReadOnlyCollection<string> KnownAdvisoriesAdded,
DateTimeOffset CompletedAt);
using StellaOps.Concelier.Connector.Common;
namespace StellaOps.Concelier.Connector.Common.State;
/// <summary>
/// Describes a raw upstream document that should be persisted for a connector during seeding.
/// </summary>
public sealed record SourceStateSeedDocument
{
/// <summary>
/// Absolute source URI. Must match the connector's upstream document identifier.
/// </summary>
public string Uri { get; init; } = string.Empty;
/// <summary>
/// Raw document payload. Required when creating or replacing a document.
/// </summary>
public byte[] Content { get; init; } = Array.Empty<byte>();
/// <summary>
/// Optional explicit document identifier. When provided it overrides auto-generated IDs.
/// </summary>
public Guid? DocumentId { get; init; }
/// <summary>
/// MIME type for the document payload.
/// </summary>
public string? ContentType { get; init; }
/// <summary>
/// Status assigned to the document. Defaults to <see cref="DocumentStatuses.PendingParse"/>.
/// </summary>
public string Status { get; init; } = DocumentStatuses.PendingParse;
/// <summary>
/// Optional HTTP-style headers persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Headers { get; init; }
/// <summary>
/// Source metadata (connector specific) persisted alongside the raw document.
/// </summary>
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
/// <summary>
/// Upstream ETag value, if available.
/// </summary>
public string? Etag { get; init; }
/// <summary>
/// Upstream last-modified timestamp, if available.
/// </summary>
public DateTimeOffset? LastModified { get; init; }
/// <summary>
/// Optional document expiration. When set a TTL will purge the raw payload after the configured retention.
/// </summary>
public DateTimeOffset? ExpiresAt { get; init; }
/// <summary>
/// Fetch timestamp stamped onto the document. Defaults to the seed completion timestamp.
/// </summary>
public DateTimeOffset? FetchedAt { get; init; }
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingDocuments</c> set.
/// </summary>
public bool AddToPendingDocuments { get; init; } = true;
/// <summary>
/// When true, the document ID will be appended to the connector cursor's <c>pendingMappings</c> set.
/// </summary>
public bool AddToPendingMappings { get; init; }
/// <summary>
/// Optional identifiers that should be recorded on the cursor to avoid duplicate ingestion.
/// </summary>
public IReadOnlyCollection<string>? KnownIdentifiers { get; init; }
}
/// <summary>
/// Cursor updates that should accompany seeded documents.
/// </summary>
public sealed record SourceStateSeedCursor
{
/// <summary>
/// Optional <c>pendingDocuments</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingDocuments { get; init; }
/// <summary>
/// Optional <c>pendingMappings</c> additions expressed as document IDs.
/// </summary>
public IReadOnlyCollection<Guid>? PendingMappings { get; init; }
/// <summary>
/// Optional known advisory identifiers to merge with the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Upstream window watermark tracked by connectors that rely on last-modified cursors.
/// </summary>
public DateTimeOffset? LastModifiedCursor { get; init; }
/// <summary>
/// Optional fetch timestamp used by connectors that track the last polling instant.
/// </summary>
public DateTimeOffset? LastFetchAt { get; init; }
/// <summary>
/// Additional cursor fields (string values) to merge.
/// </summary>
public IReadOnlyDictionary<string, string>? Additional { get; init; }
}
/// <summary>
/// Seeding specification describing the source, documents, and cursor edits to apply.
/// </summary>
public sealed record SourceStateSeedSpecification
{
/// <summary>
/// Source/connector name (e.g. <c>vndr.msrc</c>).
/// </summary>
public string Source { get; init; } = string.Empty;
/// <summary>
/// Documents that should be inserted or replaced before the cursor update.
/// </summary>
public IReadOnlyList<SourceStateSeedDocument> Documents { get; init; } = Array.Empty<SourceStateSeedDocument>();
/// <summary>
/// Cursor adjustments applied after documents are persisted.
/// </summary>
public SourceStateSeedCursor? Cursor { get; init; }
/// <summary>
/// Connector-level known advisory identifiers to merge into the cursor.
/// </summary>
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
/// <summary>
/// Optional completion timestamp. Defaults to the processor's time provider.
/// </summary>
public DateTimeOffset? CompletedAt { get; init; }
}
/// <summary>
/// Result returned after seeding completes.
/// </summary>
public sealed record SourceStateSeedResult(
int DocumentsProcessed,
int PendingDocumentsAdded,
int PendingMappingsAdded,
IReadOnlyCollection<Guid> DocumentIds,
IReadOnlyCollection<Guid> PendingDocumentIds,
IReadOnlyCollection<Guid> PendingMappingIds,
IReadOnlyCollection<string> KnownAdvisoriesAdded,
DateTimeOffset CompletedAt);

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs
View File

@@ -1,6 +1,6 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common.Fetch;
using MongoContracts = StellaOps.Concelier.Storage;
using StellaOps.Cryptography;
@@ -62,7 +62,7 @@ public sealed class SourceStateSeedProcessor
}
var state = await _stateRepository.TryGetAsync(specification.Source, cancellationToken).ConfigureAwait(false);
var cursor = state?.Cursor ?? new BsonDocument();
var cursor = state?.Cursor ?? new DocumentObject();
var newlyPendingDocuments = MergeGuidArray(cursor, "pendingDocuments", pendingDocumentIds);
var newlyPendingMappings = MergeGuidArray(cursor, "pendingMappings", pendingMappingIds);
@@ -216,14 +216,14 @@ public sealed class SourceStateSeedProcessor
return new Dictionary<string, string>(values, StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyCollection<Guid> MergeGuidArray(BsonDocument cursor, string field, IReadOnlyCollection<Guid> additions)
private static IReadOnlyCollection<Guid> MergeGuidArray(DocumentObject cursor, string field, IReadOnlyCollection<Guid> additions)
{
if (additions.Count == 0)
{
return Array.Empty<Guid>();
}
var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray
var existing = cursor.TryGetValue(field, out var value) && value is DocumentArray existingArray
? existingArray.Select(AsGuid).Where(static g => g != Guid.Empty).ToHashSet()
: new HashSet<Guid>();
@@ -243,7 +243,7 @@ public sealed class SourceStateSeedProcessor
if (existing.Count > 0)
{
cursor[field] = new BsonArray(existing
cursor[field] = new DocumentArray(existing
.Select(static g => g.ToString("D"))
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
}
@@ -251,14 +251,14 @@ public sealed class SourceStateSeedProcessor
return newlyAdded.AsReadOnly();
}
private static IReadOnlyCollection<string> MergeStringArray(BsonDocument cursor, string field, IReadOnlyCollection<string> additions)
private static IReadOnlyCollection<string> MergeStringArray(DocumentObject cursor, string field, IReadOnlyCollection<string> additions)
{
if (additions.Count == 0)
{
return Array.Empty<string>();
}
var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray
var existing = cursor.TryGetValue(field, out var value) && value is DocumentArray existingArray
? existingArray.Select(static v => v?.AsString ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.ToHashSet(StringComparer.OrdinalIgnoreCase)
@@ -281,14 +281,14 @@ public sealed class SourceStateSeedProcessor
if (existing.Count > 0)
{
cursor[field] = new BsonArray(existing
cursor[field] = new DocumentArray(existing
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
}
return newlyAdded.AsReadOnly();
}
private static Guid AsGuid(BsonValue value)
private static Guid AsGuid(DocumentValue value)
{
if (value is null)
{

214
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Telemetry/SourceDiagnostics.cs
View File

@@ -1,107 +1,107 @@
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Telemetry;
/// <summary>
/// Central telemetry instrumentation for connector HTTP operations.
/// </summary>
public static class SourceDiagnostics
{
public const string ActivitySourceName = "StellaOps.Concelier.Connector";
public const string MeterName = "StellaOps.Concelier.Connector";
private static readonly ActivitySource ActivitySource = new(ActivitySourceName);
private static readonly Meter Meter = new(MeterName);
private static readonly Counter<long> HttpRequestCounter = Meter.CreateCounter<long>("concelier.source.http.requests");
private static readonly Counter<long> HttpRetryCounter = Meter.CreateCounter<long>("concelier.source.http.retries");
private static readonly Counter<long> HttpFailureCounter = Meter.CreateCounter<long>("concelier.source.http.failures");
private static readonly Counter<long> HttpNotModifiedCounter = Meter.CreateCounter<long>("concelier.source.http.not_modified");
private static readonly Histogram<double> HttpDuration = Meter.CreateHistogram<double>("concelier.source.http.duration", unit: "ms");
private static readonly Histogram<long> HttpPayloadBytes = Meter.CreateHistogram<long>("concelier.source.http.payload_bytes", unit: "byte");
public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName)
{
var tags = new ActivityTagsCollection
{
{ "concelier.source", sourceName },
{ "http.method", httpMethod },
{ "http.url", requestUri.ToString() },
};
if (!string.IsNullOrWhiteSpace(clientName))
{
tags.Add("http.client_name", clientName!);
}
return ActivitySource.StartActivity("SourceFetch", ActivityKind.Client, parentContext: default, tags: tags);
}
public static void RecordHttpRequest(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount, TimeSpan duration, long? contentLength, string? rateLimitRemaining)
{
var tags = BuildDefaultTags(sourceName, clientName, statusCode, attemptCount);
HttpRequestCounter.Add(1, tags);
HttpDuration.Record(duration.TotalMilliseconds, tags);
if (contentLength.HasValue && contentLength.Value >= 0)
{
HttpPayloadBytes.Record(contentLength.Value, tags);
}
if (statusCode == HttpStatusCode.NotModified)
{
HttpNotModifiedCounter.Add(1, tags);
}
if ((int)statusCode >= 500 || statusCode == HttpStatusCode.TooManyRequests)
{
HttpFailureCounter.Add(1, tags);
}
if (!string.IsNullOrWhiteSpace(rateLimitRemaining) && long.TryParse(rateLimitRemaining, out var remaining))
{
tags.Add("http.rate_limit.remaining", remaining);
}
}
public static void RecordRetry(string sourceName, string? clientName, HttpStatusCode? statusCode, int attempt, TimeSpan delay)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.retry_attempt", attempt },
{ "http.retry_delay_ms", delay.TotalMilliseconds },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
if (statusCode.HasValue)
{
tags.Add("http.status_code", (int)statusCode.Value);
}
HttpRetryCounter.Add(1, tags);
}
private static TagList BuildDefaultTags(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.status_code", (int)statusCode },
{ "http.attempts", attemptCount },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
return tags;
}
}
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Net;
namespace StellaOps.Concelier.Connector.Common.Telemetry;
/// <summary>
/// Central telemetry instrumentation for connector HTTP operations.
/// </summary>
public static class SourceDiagnostics
{
public const string ActivitySourceName = "StellaOps.Concelier.Connector";
public const string MeterName = "StellaOps.Concelier.Connector";
private static readonly ActivitySource ActivitySource = new(ActivitySourceName);
private static readonly Meter Meter = new(MeterName);
private static readonly Counter<long> HttpRequestCounter = Meter.CreateCounter<long>("concelier.source.http.requests");
private static readonly Counter<long> HttpRetryCounter = Meter.CreateCounter<long>("concelier.source.http.retries");
private static readonly Counter<long> HttpFailureCounter = Meter.CreateCounter<long>("concelier.source.http.failures");
private static readonly Counter<long> HttpNotModifiedCounter = Meter.CreateCounter<long>("concelier.source.http.not_modified");
private static readonly Histogram<double> HttpDuration = Meter.CreateHistogram<double>("concelier.source.http.duration", unit: "ms");
private static readonly Histogram<long> HttpPayloadBytes = Meter.CreateHistogram<long>("concelier.source.http.payload_bytes", unit: "byte");
public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName)
{
var tags = new ActivityTagsCollection
{
{ "concelier.source", sourceName },
{ "http.method", httpMethod },
{ "http.url", requestUri.ToString() },
};
if (!string.IsNullOrWhiteSpace(clientName))
{
tags.Add("http.client_name", clientName!);
}
return ActivitySource.StartActivity("SourceFetch", ActivityKind.Client, parentContext: default, tags: tags);
}
public static void RecordHttpRequest(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount, TimeSpan duration, long? contentLength, string? rateLimitRemaining)
{
var tags = BuildDefaultTags(sourceName, clientName, statusCode, attemptCount);
HttpRequestCounter.Add(1, tags);
HttpDuration.Record(duration.TotalMilliseconds, tags);
if (contentLength.HasValue && contentLength.Value >= 0)
{
HttpPayloadBytes.Record(contentLength.Value, tags);
}
if (statusCode == HttpStatusCode.NotModified)
{
HttpNotModifiedCounter.Add(1, tags);
}
if ((int)statusCode >= 500 || statusCode == HttpStatusCode.TooManyRequests)
{
HttpFailureCounter.Add(1, tags);
}
if (!string.IsNullOrWhiteSpace(rateLimitRemaining) && long.TryParse(rateLimitRemaining, out var remaining))
{
tags.Add("http.rate_limit.remaining", remaining);
}
}
public static void RecordRetry(string sourceName, string? clientName, HttpStatusCode? statusCode, int attempt, TimeSpan delay)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.retry_attempt", attempt },
{ "http.retry_delay_ms", delay.TotalMilliseconds },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
if (statusCode.HasValue)
{
tags.Add("http.status_code", (int)statusCode.Value);
}
HttpRetryCounter.Add(1, tags);
}
private static TagList BuildDefaultTags(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount)
{
var tags = new TagList
{
{ "concelier.source", sourceName },
{ "http.status_code", (int)statusCode },
{ "http.attempts", attemptCount },
};
if (clientName is not null)
{
tags.Add("http.client_name", clientName);
}
return tags;
}
}

420
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Testing/CannedHttpMessageHandler.cs
View File

@@ -1,210 +1,210 @@
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http;
using System.Text;
namespace StellaOps.Concelier.Connector.Common.Testing;
/// <summary>
/// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method.
/// Tracks requests for assertions and supports fallbacks/exceptions.
/// </summary>
public sealed class CannedHttpMessageHandler : HttpMessageHandler
{
private readonly ConcurrentDictionary<RequestKey, ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>> _responses =
new(RequestKeyComparer.Instance);
private readonly ConcurrentQueue<CannedRequestRecord> _requests = new();
private Func<HttpRequestMessage, HttpResponseMessage>? _fallback;
/// <summary>
/// Recorded requests in arrival order.
/// </summary>
public IReadOnlyCollection<CannedRequestRecord> Requests => _requests.ToArray();
/// <summary>
/// Registers a canned response for a GET request to <paramref name="requestUri"/>.
/// </summary>
public void AddResponse(Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(HttpMethod.Get, requestUri, _ => factory());
/// <summary>
/// Registers a canned response for the specified method and URI.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(method, requestUri, _ => factory());
/// <summary>
/// Registers a canned response using the full request context.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpRequestMessage, HttpResponseMessage> factory)
{
ArgumentNullException.ThrowIfNull(method);
ArgumentNullException.ThrowIfNull(requestUri);
ArgumentNullException.ThrowIfNull(factory);
var key = new RequestKey(method, requestUri);
var queue = _responses.GetOrAdd(key, static _ => new ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>());
queue.Enqueue(factory);
}
/// <summary>
/// Registers an exception to be thrown for the specified request.
/// </summary>
public void AddException(HttpMethod method, Uri requestUri, Exception exception)
{
ArgumentNullException.ThrowIfNull(exception);
AddResponse(method, requestUri, _ => throw exception);
}
/// <summary>
/// Registers a fallback used when no specific response is queued for a request.
/// </summary>
public void SetFallback(Func<HttpRequestMessage, HttpResponseMessage> fallback)
{
ArgumentNullException.ThrowIfNull(fallback);
_fallback = fallback;
}
/// <summary>
/// Clears registered responses and captured requests.
/// </summary>
public void Clear()
{
_responses.Clear();
while (_requests.TryDequeue(out _))
{
}
_fallback = null;
}
/// <summary>
/// Throws if any responses remain queued.
/// </summary>
public void AssertNoPendingResponses()
{
foreach (var queue in _responses.Values)
{
if (!queue.IsEmpty)
{
throw new InvalidOperationException("Not all canned responses were consumed.");
}
}
}
/// <summary>
/// Creates an <see cref="HttpClient"/> wired to this handler.
/// </summary>
public HttpClient CreateClient()
=> new(this, disposeHandler: false)
{
Timeout = TimeSpan.FromSeconds(10),
};
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri is null)
{
throw new InvalidOperationException("Request URI is required for canned responses.");
}
var key = new RequestKey(request.Method ?? HttpMethod.Get, request.RequestUri);
var factory = DequeueFactory(key);
if (factory is null)
{
if (_fallback is null)
{
throw new InvalidOperationException($"No canned response registered for {request.Method} {request.RequestUri}.");
}
factory = _fallback;
}
var snapshot = CaptureRequest(request);
_requests.Enqueue(snapshot);
var response = factory(request);
response.RequestMessage ??= request;
return Task.FromResult(response);
}
private Func<HttpRequestMessage, HttpResponseMessage>? DequeueFactory(RequestKey key)
{
if (_responses.TryGetValue(key, out var queue) && queue.TryDequeue(out var factory))
{
return factory;
}
return null;
}
private static CannedRequestRecord CaptureRequest(HttpRequestMessage request)
{
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var header in request.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
if (request.Content is not null)
{
foreach (var header in request.Content.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
}
return new CannedRequestRecord(
Timestamp: DateTimeOffset.UtcNow,
Method: request.Method ?? HttpMethod.Get,
Uri: request.RequestUri!,
Headers: headers);
}
private readonly record struct RequestKey(HttpMethod Method, string Uri)
{
public RequestKey(HttpMethod method, Uri uri)
: this(method, uri.ToString())
{
}
public bool Equals(RequestKey other)
=> string.Equals(Method.Method, other.Method.Method, StringComparison.OrdinalIgnoreCase)
&& string.Equals(Uri, other.Uri, StringComparison.OrdinalIgnoreCase);
public override int GetHashCode()
{
var methodHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Method.Method);
var uriHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Uri);
return HashCode.Combine(methodHash, uriHash);
}
}
private sealed class RequestKeyComparer : IEqualityComparer<RequestKey>
{
public static readonly RequestKeyComparer Instance = new();
public bool Equals(RequestKey x, RequestKey y) => x.Equals(y);
public int GetHashCode(RequestKey obj) => obj.GetHashCode();
}
public readonly record struct CannedRequestRecord(DateTimeOffset Timestamp, HttpMethod Method, Uri Uri, IReadOnlyDictionary<string, string> Headers);
private static HttpResponseMessage BuildTextResponse(HttpStatusCode statusCode, string content, string contentType)
{
var message = new HttpResponseMessage(statusCode)
{
Content = new StringContent(content, Encoding.UTF8, contentType),
};
return message;
}
public void AddJsonResponse(Uri requestUri, string json, HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, json, "application/json"));
public void AddTextResponse(Uri requestUri, string content, string contentType = "text/plain", HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, content, contentType));
}
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http;
using System.Text;
namespace StellaOps.Concelier.Connector.Common.Testing;
/// <summary>
/// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method.
/// Tracks requests for assertions and supports fallbacks/exceptions.
/// </summary>
public sealed class CannedHttpMessageHandler : HttpMessageHandler
{
private readonly ConcurrentDictionary<RequestKey, ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>> _responses =
new(RequestKeyComparer.Instance);
private readonly ConcurrentQueue<CannedRequestRecord> _requests = new();
private Func<HttpRequestMessage, HttpResponseMessage>? _fallback;
/// <summary>
/// Recorded requests in arrival order.
/// </summary>
public IReadOnlyCollection<CannedRequestRecord> Requests => _requests.ToArray();
/// <summary>
/// Registers a canned response for a GET request to <paramref name="requestUri"/>.
/// </summary>
public void AddResponse(Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(HttpMethod.Get, requestUri, _ => factory());
/// <summary>
/// Registers a canned response for the specified method and URI.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpResponseMessage> factory)
=> AddResponse(method, requestUri, _ => factory());
/// <summary>
/// Registers a canned response using the full request context.
/// </summary>
public void AddResponse(HttpMethod method, Uri requestUri, Func<HttpRequestMessage, HttpResponseMessage> factory)
{
ArgumentNullException.ThrowIfNull(method);
ArgumentNullException.ThrowIfNull(requestUri);
ArgumentNullException.ThrowIfNull(factory);
var key = new RequestKey(method, requestUri);
var queue = _responses.GetOrAdd(key, static _ => new ConcurrentQueue<Func<HttpRequestMessage, HttpResponseMessage>>());
queue.Enqueue(factory);
}
/// <summary>
/// Registers an exception to be thrown for the specified request.
/// </summary>
public void AddException(HttpMethod method, Uri requestUri, Exception exception)
{
ArgumentNullException.ThrowIfNull(exception);
AddResponse(method, requestUri, _ => throw exception);
}
/// <summary>
/// Registers a fallback used when no specific response is queued for a request.
/// </summary>
public void SetFallback(Func<HttpRequestMessage, HttpResponseMessage> fallback)
{
ArgumentNullException.ThrowIfNull(fallback);
_fallback = fallback;
}
/// <summary>
/// Clears registered responses and captured requests.
/// </summary>
public void Clear()
{
_responses.Clear();
while (_requests.TryDequeue(out _))
{
}
_fallback = null;
}
/// <summary>
/// Throws if any responses remain queued.
/// </summary>
public void AssertNoPendingResponses()
{
foreach (var queue in _responses.Values)
{
if (!queue.IsEmpty)
{
throw new InvalidOperationException("Not all canned responses were consumed.");
}
}
}
/// <summary>
/// Creates an <see cref="HttpClient"/> wired to this handler.
/// </summary>
public HttpClient CreateClient()
=> new(this, disposeHandler: false)
{
Timeout = TimeSpan.FromSeconds(10),
};
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri is null)
{
throw new InvalidOperationException("Request URI is required for canned responses.");
}
var key = new RequestKey(request.Method ?? HttpMethod.Get, request.RequestUri);
var factory = DequeueFactory(key);
if (factory is null)
{
if (_fallback is null)
{
throw new InvalidOperationException($"No canned response registered for {request.Method} {request.RequestUri}.");
}
factory = _fallback;
}
var snapshot = CaptureRequest(request);
_requests.Enqueue(snapshot);
var response = factory(request);
response.RequestMessage ??= request;
return Task.FromResult(response);
}
private Func<HttpRequestMessage, HttpResponseMessage>? DequeueFactory(RequestKey key)
{
if (_responses.TryGetValue(key, out var queue) && queue.TryDequeue(out var factory))
{
return factory;
}
return null;
}
private static CannedRequestRecord CaptureRequest(HttpRequestMessage request)
{
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var header in request.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
if (request.Content is not null)
{
foreach (var header in request.Content.Headers)
{
headers[header.Key] = string.Join(',', header.Value);
}
}
return new CannedRequestRecord(
Timestamp: DateTimeOffset.UtcNow,
Method: request.Method ?? HttpMethod.Get,
Uri: request.RequestUri!,
Headers: headers);
}
private readonly record struct RequestKey(HttpMethod Method, string Uri)
{
public RequestKey(HttpMethod method, Uri uri)
: this(method, uri.ToString())
{
}
public bool Equals(RequestKey other)
=> string.Equals(Method.Method, other.Method.Method, StringComparison.OrdinalIgnoreCase)
&& string.Equals(Uri, other.Uri, StringComparison.OrdinalIgnoreCase);
public override int GetHashCode()
{
var methodHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Method.Method);
var uriHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Uri);
return HashCode.Combine(methodHash, uriHash);
}
}
private sealed class RequestKeyComparer : IEqualityComparer<RequestKey>
{
public static readonly RequestKeyComparer Instance = new();
public bool Equals(RequestKey x, RequestKey y) => x.Equals(y);
public int GetHashCode(RequestKey obj) => obj.GetHashCode();
}
public readonly record struct CannedRequestRecord(DateTimeOffset Timestamp, HttpMethod Method, Uri Uri, IReadOnlyDictionary<string, string> Headers);
private static HttpResponseMessage BuildTextResponse(HttpStatusCode statusCode, string content, string contentType)
{
var message = new HttpResponseMessage(statusCode)
{
Content = new StringContent(content, Encoding.UTF8, contentType),
};
return message;
}
public void AddJsonResponse(Uri requestUri, string json, HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, json, "application/json"));
public void AddTextResponse(Uri requestUri, string content, string contentType = "text/plain", HttpStatusCode statusCode = HttpStatusCode.OK)
=> AddResponse(requestUri, () => BuildTextResponse(statusCode, content, contentType));
}

124
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Url/UrlNormalizer.cs
View File

@@ -1,62 +1,62 @@
namespace StellaOps.Concelier.Connector.Common.Url;
/// <summary>
/// Utilities for normalizing URLs from upstream feeds.
/// </summary>
public static class UrlNormalizer
{
/// <summary>
/// Attempts to normalize <paramref name="value"/> relative to <paramref name="baseUri"/>.
/// Removes fragments and enforces HTTPS when possible.
/// </summary>
public static bool TryNormalize(string? value, Uri? baseUri, out Uri? normalized, bool stripFragment = true, bool forceHttps = false)
{
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!Uri.TryCreate(value.Trim(), UriKind.RelativeOrAbsolute, out var candidate))
{
return false;
}
if (!candidate.IsAbsoluteUri)
{
if (baseUri is null)
{
return false;
}
if (!Uri.TryCreate(baseUri, candidate, out candidate))
{
return false;
}
}
if (forceHttps && string.Equals(candidate.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase))
{
candidate = new UriBuilder(candidate) { Scheme = Uri.UriSchemeHttps, Port = candidate.IsDefaultPort ? -1 : candidate.Port }.Uri;
}
if (stripFragment && !string.IsNullOrEmpty(candidate.Fragment))
{
var builder = new UriBuilder(candidate) { Fragment = string.Empty };
candidate = builder.Uri;
}
normalized = candidate;
return true;
}
public static Uri NormalizeOrThrow(string value, Uri? baseUri = null, bool stripFragment = true, bool forceHttps = false)
{
if (!TryNormalize(value, baseUri, out var normalized, stripFragment, forceHttps) || normalized is null)
{
throw new FormatException($"Value '{value}' is not a valid URI");
}
return normalized;
}
}
namespace StellaOps.Concelier.Connector.Common.Url;
/// <summary>
/// Utilities for normalizing URLs from upstream feeds.
/// </summary>
public static class UrlNormalizer
{
/// <summary>
/// Attempts to normalize <paramref name="value"/> relative to <paramref name="baseUri"/>.
/// Removes fragments and enforces HTTPS when possible.
/// </summary>
public static bool TryNormalize(string? value, Uri? baseUri, out Uri? normalized, bool stripFragment = true, bool forceHttps = false)
{
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
if (!Uri.TryCreate(value.Trim(), UriKind.RelativeOrAbsolute, out var candidate))
{
return false;
}
if (!candidate.IsAbsoluteUri)
{
if (baseUri is null)
{
return false;
}
if (!Uri.TryCreate(baseUri, candidate, out candidate))
{
return false;
}
}
if (forceHttps && string.Equals(candidate.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase))
{
candidate = new UriBuilder(candidate) { Scheme = Uri.UriSchemeHttps, Port = candidate.IsDefaultPort ? -1 : candidate.Port }.Uri;
}
if (stripFragment && !string.IsNullOrEmpty(candidate.Fragment))
{
var builder = new UriBuilder(candidate) { Fragment = string.Empty };
candidate = builder.Uri;
}
normalized = candidate;
return true;
}
public static Uri NormalizeOrThrow(string value, Uri? baseUri = null, bool stripFragment = true, bool forceHttps = false)
{
if (!TryNormalize(value, baseUri, out var normalized, stripFragment, forceHttps) || normalized is null)
{
throw new FormatException($"Value '{value}' is not a valid URI");
}
return normalized;
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Xml/IXmlSchemaValidator.cs
View File

@@ -1,9 +1,9 @@
using System.Xml.Linq;
using System.Xml.Schema;
namespace StellaOps.Concelier.Connector.Common.Xml;
public interface IXmlSchemaValidator
{
void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName);
}
using System.Xml.Linq;
using System.Xml.Schema;
namespace StellaOps.Concelier.Connector.Common.Xml;
public interface IXmlSchemaValidator
{
void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName);
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationError.cs
View File

@@ -1,3 +1,3 @@
namespace StellaOps.Concelier.Connector.Common.Xml;
public sealed record XmlSchemaValidationError(string Message, string? Location);
namespace StellaOps.Concelier.Connector.Common.Xml;
public sealed record XmlSchemaValidationError(string Message, string? Location);

36
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationException.cs
View File

@@ -1,18 +1,18 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Common.Xml;
public sealed class XmlSchemaValidationException : Exception
{
public XmlSchemaValidationException(string documentName, IReadOnlyList<XmlSchemaValidationError> errors)
: base($"XML schema validation failed for '{documentName}'.")
{
DocumentName = documentName;
Errors = errors ?? Array.Empty<XmlSchemaValidationError>();
}
public string DocumentName { get; }
public IReadOnlyList<XmlSchemaValidationError> Errors { get; }
}
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Common.Xml;
public sealed class XmlSchemaValidationException : Exception
{
public XmlSchemaValidationException(string documentName, IReadOnlyList<XmlSchemaValidationError> errors)
: base($"XML schema validation failed for '{documentName}'.")
{
DocumentName = documentName;
Errors = errors ?? Array.Empty<XmlSchemaValidationError>();
}
public string DocumentName { get; }
public IReadOnlyList<XmlSchemaValidationError> Errors { get; }
}

142
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidator.cs
View File

@@ -1,71 +1,71 @@
using System;
using System.Collections.Generic;
using System.Xml.Linq;
using System.Xml.Schema;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Xml;
public sealed class XmlSchemaValidator : IXmlSchemaValidator
{
private readonly ILogger<XmlSchemaValidator> _logger;
public XmlSchemaValidator(ILogger<XmlSchemaValidator> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(schemaSet);
ArgumentException.ThrowIfNullOrWhiteSpace(documentName);
var errors = new List<XmlSchemaValidationError>();
void Handler(object? sender, ValidationEventArgs args)
{
if (args is null)
{
return;
}
var location = FormatLocation(args.Exception);
errors.Add(new XmlSchemaValidationError(args.Message, location));
}
try
{
document.Validate(schemaSet, Handler, addSchemaInfo: true);
}
catch (System.Xml.Schema.XmlSchemaValidationException ex)
{
var location = FormatLocation(ex);
errors.Add(new XmlSchemaValidationError(ex.Message, location));
}
if (errors.Count > 0)
{
var exception = new XmlSchemaValidationException(documentName, errors);
_logger.LogError(exception, "XML schema validation failed for {DocumentName}", documentName);
throw exception;
}
_logger.LogDebug("XML schema validation succeeded for {DocumentName}", documentName);
}
private static string? FormatLocation(System.Xml.Schema.XmlSchemaException? exception)
{
if (exception is null)
{
return null;
}
if (exception.LineNumber <= 0)
{
return null;
}
return $"line {exception.LineNumber}, position {exception.LinePosition}";
}
}
using System;
using System.Collections.Generic;
using System.Xml.Linq;
using System.Xml.Schema;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Common.Xml;
public sealed class XmlSchemaValidator : IXmlSchemaValidator
{
private readonly ILogger<XmlSchemaValidator> _logger;
public XmlSchemaValidator(ILogger<XmlSchemaValidator> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(schemaSet);
ArgumentException.ThrowIfNullOrWhiteSpace(documentName);
var errors = new List<XmlSchemaValidationError>();
void Handler(object? sender, ValidationEventArgs args)
{
if (args is null)
{
return;
}
var location = FormatLocation(args.Exception);
errors.Add(new XmlSchemaValidationError(args.Message, location));
}
try
{
document.Validate(schemaSet, Handler, addSchemaInfo: true);
}
catch (System.Xml.Schema.XmlSchemaValidationException ex)
{
var location = FormatLocation(ex);
errors.Add(new XmlSchemaValidationError(ex.Message, location));
}
if (errors.Count > 0)
{
var exception = new XmlSchemaValidationException(documentName, errors);
_logger.LogError(exception, "XML schema validation failed for {DocumentName}", documentName);
throw exception;
}
_logger.LogDebug("XML schema validation succeeded for {DocumentName}", documentName);
}
private static string? FormatLocation(System.Xml.Schema.XmlSchemaException? exception)
{
if (exception is null)
{
return null;
}
if (exception.LineNumber <= 0)
{
return null;
}
return $"line {exception.LineNumber}, position {exception.LinePosition}";
}
}