Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,43 @@
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays.
|
||||
/// </summary>
|
||||
public sealed class CryptoJitterSource : IJitterSource
|
||||
{
|
||||
public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive)
|
||||
{
|
||||
if (maxInclusive < minInclusive)
|
||||
{
|
||||
throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive));
|
||||
}
|
||||
|
||||
if (minInclusive < TimeSpan.Zero)
|
||||
{
|
||||
minInclusive = TimeSpan.Zero;
|
||||
}
|
||||
|
||||
if (maxInclusive == minInclusive)
|
||||
{
|
||||
return minInclusive;
|
||||
}
|
||||
|
||||
var minTicks = minInclusive.Ticks;
|
||||
var maxTicks = maxInclusive.Ticks;
|
||||
var range = maxTicks - minTicks;
|
||||
|
||||
Span<byte> buffer = stackalloc byte[8];
|
||||
RandomNumberGenerator.Fill(buffer);
|
||||
var sample = BitConverter.ToUInt64(buffer);
|
||||
var ratio = sample / (double)ulong.MaxValue;
|
||||
var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero);
|
||||
if (jitterTicks > range)
|
||||
{
|
||||
jitterTicks = range;
|
||||
}
|
||||
|
||||
return TimeSpan.FromTicks(minTicks + jitterTicks);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Produces random jitter durations used to decorrelate retries.
|
||||
/// </summary>
|
||||
public interface IJitterSource
|
||||
{
|
||||
TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive);
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.GridFS;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Handles persistence of raw upstream documents in GridFS buckets for later parsing.
|
||||
/// </summary>
|
||||
public sealed class RawDocumentStorage
|
||||
{
|
||||
private const string BucketName = "documents";
|
||||
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public RawDocumentStorage(IMongoDatabase database)
|
||||
{
|
||||
_database = database ?? throw new ArgumentNullException(nameof(database));
|
||||
}
|
||||
|
||||
private GridFSBucket CreateBucket() => new(_database, new GridFSBucketOptions
|
||||
{
|
||||
BucketName = BucketName,
|
||||
WriteConcern = _database.Settings.WriteConcern,
|
||||
ReadConcern = _database.Settings.ReadConcern,
|
||||
});
|
||||
|
||||
public Task<ObjectId> UploadAsync(
|
||||
string sourceName,
|
||||
string uri,
|
||||
byte[] content,
|
||||
string? contentType,
|
||||
CancellationToken cancellationToken)
|
||||
=> UploadAsync(sourceName, uri, content, contentType, expiresAt: null, cancellationToken);
|
||||
|
||||
public async Task<ObjectId> UploadAsync(
|
||||
string sourceName,
|
||||
string uri,
|
||||
byte[] content,
|
||||
string? contentType,
|
||||
DateTimeOffset? expiresAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(sourceName);
|
||||
ArgumentException.ThrowIfNullOrEmpty(uri);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var bucket = CreateBucket();
|
||||
var filename = $"{sourceName}/{Guid.NewGuid():N}";
|
||||
var metadata = new BsonDocument
|
||||
{
|
||||
["sourceName"] = sourceName,
|
||||
["uri"] = uri,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(contentType))
|
||||
{
|
||||
metadata["contentType"] = contentType;
|
||||
}
|
||||
|
||||
if (expiresAt.HasValue)
|
||||
{
|
||||
metadata["expiresAt"] = expiresAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return await bucket.UploadFromBytesAsync(filename, content, new GridFSUploadOptions
|
||||
{
|
||||
Metadata = metadata,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public Task<byte[]> DownloadAsync(ObjectId id, CancellationToken cancellationToken)
|
||||
{
|
||||
var bucket = CreateBucket();
|
||||
return bucket.DownloadAsBytesAsync(id, cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DeleteAsync(ObjectId id, CancellationToken cancellationToken)
|
||||
{
|
||||
var bucket = CreateBucket();
|
||||
try
|
||||
{
|
||||
await bucket.DeleteAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (GridFSFileNotFoundException)
|
||||
{
|
||||
// Already removed; ignore.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Result of fetching raw response content without persisting a document.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchContentResult
|
||||
{
|
||||
private SourceFetchContentResult(
|
||||
HttpStatusCode statusCode,
|
||||
byte[]? content,
|
||||
bool notModified,
|
||||
string? etag,
|
||||
DateTimeOffset? lastModified,
|
||||
string? contentType,
|
||||
int attempts,
|
||||
IReadOnlyDictionary<string, string>? headers)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
Content = content;
|
||||
IsNotModified = notModified;
|
||||
ETag = etag;
|
||||
LastModified = lastModified;
|
||||
ContentType = contentType;
|
||||
Attempts = attempts;
|
||||
Headers = headers;
|
||||
}
|
||||
|
||||
public HttpStatusCode StatusCode { get; }
|
||||
|
||||
public byte[]? Content { get; }
|
||||
|
||||
public bool IsSuccess => Content is not null;
|
||||
|
||||
public bool IsNotModified { get; }
|
||||
|
||||
public string? ETag { get; }
|
||||
|
||||
public DateTimeOffset? LastModified { get; }
|
||||
|
||||
public string? ContentType { get; }
|
||||
|
||||
public int Attempts { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string>? Headers { get; }
|
||||
|
||||
public static SourceFetchContentResult Success(
|
||||
HttpStatusCode statusCode,
|
||||
byte[] content,
|
||||
string? etag,
|
||||
DateTimeOffset? lastModified,
|
||||
string? contentType,
|
||||
int attempts,
|
||||
IReadOnlyDictionary<string, string>? headers)
|
||||
=> new(statusCode, content, notModified: false, etag, lastModified, contentType, attempts, headers);
|
||||
|
||||
public static SourceFetchContentResult NotModified(HttpStatusCode statusCode, int attempts)
|
||||
=> new(statusCode, null, notModified: true, etag: null, lastModified: null, contentType: null, attempts, headers: null);
|
||||
|
||||
public static SourceFetchContentResult Skipped(HttpStatusCode statusCode, int attempts)
|
||||
=> new(statusCode, null, notModified: false, etag: null, lastModified: null, contentType: null, attempts, headers: null);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Parameters describing a fetch operation for a source connector.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchRequest(
|
||||
string ClientName,
|
||||
string SourceName,
|
||||
HttpMethod Method,
|
||||
Uri RequestUri,
|
||||
IReadOnlyDictionary<string, string>? Metadata = null,
|
||||
string? ETag = null,
|
||||
DateTimeOffset? LastModified = null,
|
||||
TimeSpan? TimeoutOverride = null,
|
||||
IReadOnlyList<string>? AcceptHeaders = null)
|
||||
{
|
||||
public SourceFetchRequest(string clientName, string sourceName, Uri requestUri)
|
||||
: this(clientName, sourceName, HttpMethod.Get, requestUri)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
using System.Net;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of fetching a raw document from an upstream source.
|
||||
/// </summary>
|
||||
public sealed record SourceFetchResult
|
||||
{
|
||||
private SourceFetchResult(HttpStatusCode statusCode, DocumentRecord? document, bool notModified)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
Document = document;
|
||||
IsNotModified = notModified;
|
||||
}
|
||||
|
||||
public HttpStatusCode StatusCode { get; }
|
||||
|
||||
public DocumentRecord? Document { get; }
|
||||
|
||||
public bool IsSuccess => Document is not null;
|
||||
|
||||
public bool IsNotModified { get; }
|
||||
|
||||
public static SourceFetchResult Success(DocumentRecord document, HttpStatusCode statusCode)
|
||||
=> new(statusCode, document, notModified: false);
|
||||
|
||||
public static SourceFetchResult NotModified(HttpStatusCode statusCode)
|
||||
=> new(statusCode, null, notModified: true);
|
||||
|
||||
public static SourceFetchResult Skipped(HttpStatusCode statusCode)
|
||||
=> new(statusCode, null, notModified: false);
|
||||
}
|
||||
@@ -0,0 +1,726 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Telemetry;
|
||||
using StellaOps.Concelier.Core.Aoc;
|
||||
using StellaOps.Concelier.Core.Linksets;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Executes HTTP fetches for connectors, capturing raw responses with metadata for downstream stages.
|
||||
/// </summary>
|
||||
public sealed class SourceFetchService
|
||||
{
|
||||
private static readonly string[] DefaultAcceptHeaders = { "application/json" };
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly ILogger<SourceFetchService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<SourceHttpClientOptions> _httpClientOptions;
|
||||
private readonly IOptions<MongoStorageOptions> _storageOptions;
|
||||
private readonly IJitterSource _jitterSource;
|
||||
private readonly IAdvisoryRawWriteGuard _guard;
|
||||
private readonly IAdvisoryLinksetMapper _linksetMapper;
|
||||
private readonly string _connectorVersion;
|
||||
|
||||
public SourceFetchService(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
ILogger<SourceFetchService> logger,
|
||||
IJitterSource jitterSource,
|
||||
IAdvisoryRawWriteGuard guard,
|
||||
IAdvisoryLinksetMapper linksetMapper,
|
||||
TimeProvider? timeProvider = null,
|
||||
IOptionsMonitor<SourceHttpClientOptions>? httpClientOptions = null,
|
||||
IOptions<MongoStorageOptions>? storageOptions = null)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource));
|
||||
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
|
||||
_linksetMapper = linksetMapper ?? throw new ArgumentNullException(nameof(linksetMapper));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_httpClientOptions = httpClientOptions ?? throw new ArgumentNullException(nameof(httpClientOptions));
|
||||
_storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions));
|
||||
_connectorVersion = typeof(SourceFetchService).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
}
|
||||
|
||||
public async Task<SourceFetchResult> FetchAsync(SourceFetchRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
using var activity = SourceDiagnostics.StartFetch(request.SourceName, request.RequestUri, request.Method.Method, request.ClientName);
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
try
|
||||
{
|
||||
var sendResult = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
var response = sendResult.Response;
|
||||
|
||||
using (response)
|
||||
{
|
||||
var duration = stopwatch.Elapsed;
|
||||
activity?.SetTag("http.status_code", (int)response.StatusCode);
|
||||
activity?.SetTag("http.retry.count", sendResult.Attempts - 1);
|
||||
|
||||
var rateLimitRemaining = TryGetHeaderValue(response.Headers, "x-ratelimit-remaining");
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.NotModified)
|
||||
{
|
||||
_logger.LogDebug("Source {Source} returned 304 Not Modified for {Uri}", request.SourceName, request.RequestUri);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Ok);
|
||||
return SourceFetchResult.NotModified(response.StatusCode);
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var body = await ReadResponsePreviewAsync(response, cancellationToken).ConfigureAwait(false);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Error, body);
|
||||
throw new HttpRequestException($"Fetch failed with status {(int)response.StatusCode} {response.StatusCode} from {request.RequestUri}. Body preview: {body}");
|
||||
}
|
||||
|
||||
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
var contentHash = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant();
|
||||
var fetchedAt = _timeProvider.GetUtcNow();
|
||||
var contentType = response.Content.Headers.ContentType?.ToString();
|
||||
|
||||
var headers = CreateHeaderDictionary(response);
|
||||
|
||||
var metadata = request.Metadata is null
|
||||
? new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, string>(request.Metadata, StringComparer.Ordinal);
|
||||
metadata["attempts"] = sendResult.Attempts.ToString(CultureInfo.InvariantCulture);
|
||||
metadata["fetchedAt"] = fetchedAt.ToString("O");
|
||||
|
||||
var guardDocument = CreateRawAdvisoryDocument(
|
||||
request,
|
||||
response,
|
||||
contentBytes,
|
||||
contentHash,
|
||||
metadata,
|
||||
headers,
|
||||
fetchedAt);
|
||||
_guard.EnsureValid(guardDocument);
|
||||
|
||||
var storageOptions = _storageOptions.Value;
|
||||
var retention = storageOptions.RawDocumentRetention;
|
||||
DateTimeOffset? expiresAt = null;
|
||||
if (retention > TimeSpan.Zero)
|
||||
{
|
||||
var grace = storageOptions.RawDocumentRetentionTtlGrace >= TimeSpan.Zero
|
||||
? storageOptions.RawDocumentRetentionTtlGrace
|
||||
: TimeSpan.Zero;
|
||||
|
||||
try
|
||||
{
|
||||
expiresAt = fetchedAt.Add(retention).Add(grace);
|
||||
}
|
||||
catch (ArgumentOutOfRangeException)
|
||||
{
|
||||
expiresAt = DateTimeOffset.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
var gridFsId = await _rawDocumentStorage.UploadAsync(
|
||||
request.SourceName,
|
||||
request.RequestUri.ToString(),
|
||||
contentBytes,
|
||||
contentType,
|
||||
expiresAt,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
var recordId = existing?.Id ?? Guid.NewGuid();
|
||||
|
||||
var record = new DocumentRecord(
|
||||
recordId,
|
||||
request.SourceName,
|
||||
request.RequestUri.ToString(),
|
||||
fetchedAt,
|
||||
contentHash,
|
||||
DocumentStatuses.PendingParse,
|
||||
contentType,
|
||||
headers,
|
||||
metadata,
|
||||
response.Headers.ETag?.Tag,
|
||||
response.Content.Headers.LastModified,
|
||||
gridFsId,
|
||||
expiresAt);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, contentBytes.LongLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Ok);
|
||||
_logger.LogInformation("Fetched {Source} document {Uri} (sha256={Sha})", request.SourceName, request.RequestUri, contentHash);
|
||||
return SourceFetchResult.Success(upserted, response.StatusCode);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private AdvisoryRawDocument CreateRawAdvisoryDocument(
|
||||
SourceFetchRequest request,
|
||||
HttpResponseMessage response,
|
||||
byte[] contentBytes,
|
||||
string contentHash,
|
||||
IDictionary<string, string> metadata,
|
||||
IDictionary<string, string> headers,
|
||||
DateTimeOffset fetchedAt)
|
||||
{
|
||||
var tenant = _storageOptions.Value.DefaultTenant;
|
||||
|
||||
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var pair in metadata)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(pair.Key) && pair.Value is not null)
|
||||
{
|
||||
metadataBuilder[pair.Key] = pair.Value;
|
||||
}
|
||||
}
|
||||
|
||||
using var jsonDocument = ParseContent(request, contentBytes);
|
||||
var metadataSnapshot = metadataBuilder.ToImmutable();
|
||||
|
||||
var stream = ResolveStream(metadataSnapshot, response, request);
|
||||
if (!string.IsNullOrWhiteSpace(stream))
|
||||
{
|
||||
metadataBuilder["source.stream"] = stream!;
|
||||
metadataSnapshot = metadataBuilder.ToImmutable();
|
||||
}
|
||||
|
||||
var vendor = ResolveVendor(request.SourceName, metadataSnapshot);
|
||||
metadataBuilder["source.vendor"] = vendor;
|
||||
metadataBuilder["source.connector_version"] = _connectorVersion;
|
||||
|
||||
metadataSnapshot = metadataBuilder.ToImmutable();
|
||||
|
||||
var headerSnapshot = headers.ToImmutableDictionary(
|
||||
static pair => pair.Key,
|
||||
static pair => pair.Value,
|
||||
StringComparer.Ordinal);
|
||||
|
||||
var provenance = BuildProvenance(request, response, metadataSnapshot, headerSnapshot, fetchedAt, contentHash);
|
||||
|
||||
var upstreamId = ResolveUpstreamId(metadataSnapshot, request);
|
||||
var documentVersion = ResolveDocumentVersion(metadataSnapshot, response, fetchedAt);
|
||||
var signature = CreateSignatureMetadata(metadataSnapshot);
|
||||
|
||||
var aliases = upstreamId is null
|
||||
? ImmutableArray<string>.Empty
|
||||
: ImmutableArray.Create(upstreamId);
|
||||
|
||||
var identifiers = new RawIdentifiers(aliases, upstreamId ?? contentHash);
|
||||
|
||||
var contentFormat = ResolveFormat(metadataSnapshot, response);
|
||||
var specVersion = GetMetadataValue(metadataSnapshot, "content.specVersion", "content.spec_version");
|
||||
var encoding = response.Content.Headers.ContentType?.CharSet;
|
||||
|
||||
var content = new RawContent(
|
||||
contentFormat,
|
||||
specVersion,
|
||||
jsonDocument.RootElement.Clone(),
|
||||
encoding);
|
||||
|
||||
var source = new RawSourceMetadata(
|
||||
vendor,
|
||||
request.SourceName,
|
||||
_connectorVersion,
|
||||
stream);
|
||||
|
||||
var upstream = new RawUpstreamMetadata(
|
||||
upstreamId ?? request.RequestUri.ToString(),
|
||||
documentVersion,
|
||||
fetchedAt,
|
||||
contentHash,
|
||||
signature,
|
||||
provenance);
|
||||
|
||||
var supersedes = GetMetadataValue(metadataSnapshot, "supersedes");
|
||||
|
||||
var rawDocument = new AdvisoryRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
new RawLinkset
|
||||
{
|
||||
Aliases = aliases,
|
||||
PackageUrls = ImmutableArray<string>.Empty,
|
||||
Cpes = ImmutableArray<string>.Empty,
|
||||
References = ImmutableArray<RawReference>.Empty,
|
||||
ReconciledFrom = ImmutableArray<string>.Empty,
|
||||
Notes = ImmutableDictionary<string, string>.Empty
|
||||
},
|
||||
supersedes);
|
||||
|
||||
var mappedLinkset = _linksetMapper.Map(rawDocument);
|
||||
rawDocument = rawDocument with { Linkset = mappedLinkset };
|
||||
ApplyRawDocumentMetadata(metadata, rawDocument);
|
||||
return rawDocument;
|
||||
}
|
||||
|
||||
private static JsonDocument ParseContent(SourceFetchRequest request, byte[] contentBytes)
|
||||
{
|
||||
if (contentBytes is null || contentBytes.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Source {request.SourceName} returned an empty payload for {request.RequestUri}.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonDocument.Parse(contentBytes);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Raw advisory payload from {request.SourceName} is not valid JSON ({request.RequestUri}).", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static ImmutableDictionary<string, string> BuildProvenance(
|
||||
SourceFetchRequest request,
|
||||
HttpResponseMessage response,
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
ImmutableDictionary<string, string> headers,
|
||||
DateTimeOffset fetchedAt,
|
||||
string contentHash)
|
||||
{
|
||||
var builder = metadata.ToBuilder();
|
||||
foreach (var header in headers)
|
||||
{
|
||||
var key = $"http.header.{header.Key.Trim().ToLowerInvariant()}";
|
||||
builder[key] = header.Value;
|
||||
}
|
||||
|
||||
builder["http.status_code"] = ((int)response.StatusCode).ToString(CultureInfo.InvariantCulture);
|
||||
builder["http.method"] = request.Method.Method;
|
||||
builder["http.uri"] = request.RequestUri.ToString();
|
||||
|
||||
if (response.Headers.ETag?.Tag is { } etag)
|
||||
{
|
||||
builder["http.etag"] = etag;
|
||||
}
|
||||
|
||||
if (response.Content.Headers.LastModified is { } lastModified)
|
||||
{
|
||||
builder["http.last_modified"] = lastModified.ToString("O");
|
||||
}
|
||||
|
||||
builder["fetch.fetched_at"] = fetchedAt.ToString("O");
|
||||
builder["fetch.content_hash"] = contentHash;
|
||||
builder["source.client"] = request.ClientName;
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private string ResolveVendor(string sourceName, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata.TryGetValue("source.vendor", out var vendor) && !string.IsNullOrWhiteSpace(vendor))
|
||||
{
|
||||
return vendor.Trim();
|
||||
}
|
||||
|
||||
return ExtractVendorIdentifier(sourceName);
|
||||
}
|
||||
|
||||
private static string? ResolveStream(
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
HttpResponseMessage response,
|
||||
SourceFetchRequest request)
|
||||
{
|
||||
foreach (var key in new[] { "source.stream", "connector.stream", "stream" })
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(response.Content.Headers.ContentType?.MediaType))
|
||||
{
|
||||
return response.Content.Headers.ContentType!.MediaType;
|
||||
}
|
||||
|
||||
return request.RequestUri.Segments.LastOrDefault()?.Trim('/');
|
||||
}
|
||||
|
||||
private static string ResolveFormat(ImmutableDictionary<string, string> metadata, HttpResponseMessage response)
|
||||
{
|
||||
if (metadata.TryGetValue("content.format", out var format) && !string.IsNullOrWhiteSpace(format))
|
||||
{
|
||||
return format.Trim();
|
||||
}
|
||||
|
||||
return response.Content.Headers.ContentType?.MediaType ?? "unknown";
|
||||
}
|
||||
|
||||
private static string? ResolveUpstreamId(ImmutableDictionary<string, string> metadata, SourceFetchRequest request)
|
||||
{
|
||||
var candidateKeys = new[]
|
||||
{
|
||||
"aoc.upstream_id",
|
||||
"upstream.id",
|
||||
"upstreamId",
|
||||
"advisory.id",
|
||||
"advisoryId",
|
||||
"vulnerability.id",
|
||||
"vulnerabilityId",
|
||||
"cve",
|
||||
"cveId",
|
||||
"ghsa",
|
||||
"ghsaId",
|
||||
"msrc.advisoryId",
|
||||
"msrc.vulnerabilityId",
|
||||
"oracle.csaf.entryId",
|
||||
"ubuntu.advisoryId",
|
||||
"ics.advisoryId",
|
||||
"document.id",
|
||||
};
|
||||
|
||||
foreach (var key in candidateKeys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
var segments = request.RequestUri.Segments;
|
||||
if (segments.Length > 0)
|
||||
{
|
||||
var last = segments[^1].Trim('/');
|
||||
if (!string.IsNullOrEmpty(last))
|
||||
{
|
||||
return last;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? ResolveDocumentVersion(ImmutableDictionary<string, string> metadata, HttpResponseMessage response, DateTimeOffset fetchedAt)
|
||||
{
|
||||
var candidateKeys = new[]
|
||||
{
|
||||
"upstream.version",
|
||||
"document.version",
|
||||
"revision",
|
||||
"msrc.lastModified",
|
||||
"msrc.releaseDate",
|
||||
"ubuntu.version",
|
||||
"oracle.csaf.revision",
|
||||
"lastModified",
|
||||
"modified",
|
||||
"published",
|
||||
};
|
||||
|
||||
foreach (var key in candidateKeys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
if (response.Content.Headers.LastModified is { } lastModified)
|
||||
{
|
||||
return lastModified.ToString("O");
|
||||
}
|
||||
|
||||
if (response.Headers.TryGetValues("Last-Modified", out var values))
|
||||
{
|
||||
var first = values.FirstOrDefault();
|
||||
if (!string.IsNullOrWhiteSpace(first))
|
||||
{
|
||||
return first.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return fetchedAt.ToString("O");
|
||||
}
|
||||
|
||||
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (!TryGetBoolean(metadata, out var present, "upstream.signature.present", "signature.present"))
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
}
|
||||
|
||||
if (!present)
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
}
|
||||
|
||||
var format = GetMetadataValue(metadata, "upstream.signature.format", "signature.format");
|
||||
var keyId = GetMetadataValue(metadata, "upstream.signature.key_id", "signature.key_id");
|
||||
var signature = GetMetadataValue(metadata, "upstream.signature.sig", "signature.sig");
|
||||
var certificate = GetMetadataValue(metadata, "upstream.signature.certificate", "signature.certificate");
|
||||
var digest = GetMetadataValue(metadata, "upstream.signature.digest", "signature.digest");
|
||||
|
||||
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
|
||||
}
|
||||
|
||||
private static bool TryGetBoolean(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var raw) && bool.TryParse(raw, out value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string? GetMetadataValue(ImmutableDictionary<string, string> metadata, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string ExtractVendorIdentifier(string sourceName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sourceName))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var normalized = sourceName.Trim();
|
||||
var separatorIndex = normalized.LastIndexOfAny(new[] { '.', ':' });
|
||||
if (separatorIndex >= 0 && separatorIndex < normalized.Length - 1)
|
||||
{
|
||||
return normalized[(separatorIndex + 1)..];
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static void ApplyRawDocumentMetadata(IDictionary<string, string> metadata, AdvisoryRawDocument document)
|
||||
{
|
||||
metadata["tenant"] = document.Tenant;
|
||||
metadata["source.vendor"] = document.Source.Vendor;
|
||||
metadata["source.connector_version"] = document.Source.ConnectorVersion;
|
||||
if (!string.IsNullOrWhiteSpace(document.Source.Stream))
|
||||
{
|
||||
metadata["source.stream"] = document.Source.Stream!;
|
||||
}
|
||||
|
||||
metadata["upstream.upstream_id"] = document.Upstream.UpstreamId;
|
||||
metadata["upstream.content_hash"] = document.Upstream.ContentHash;
|
||||
if (!string.IsNullOrWhiteSpace(document.Upstream.DocumentVersion))
|
||||
{
|
||||
metadata["upstream.document_version"] = document.Upstream.DocumentVersion!;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<SourceFetchContentResult> FetchContentAsync(SourceFetchRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
using var activity = SourceDiagnostics.StartFetch(request.SourceName, request.RequestUri, request.Method.Method, request.ClientName);
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
try
|
||||
{
|
||||
_ = _httpClientOptions.Get(request.ClientName);
|
||||
var sendResult = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
var response = sendResult.Response;
|
||||
|
||||
using (response)
|
||||
{
|
||||
var duration = stopwatch.Elapsed;
|
||||
activity?.SetTag("http.status_code", (int)response.StatusCode);
|
||||
activity?.SetTag("http.retry.count", sendResult.Attempts - 1);
|
||||
|
||||
var rateLimitRemaining = TryGetHeaderValue(response.Headers, "x-ratelimit-remaining");
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.NotModified)
|
||||
{
|
||||
_logger.LogDebug("Source {Source} returned 304 Not Modified for {Uri}", request.SourceName, request.RequestUri);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Ok);
|
||||
return SourceFetchContentResult.NotModified(response.StatusCode, sendResult.Attempts);
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var body = await ReadResponsePreviewAsync(response, cancellationToken).ConfigureAwait(false);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Error, body);
|
||||
throw new HttpRequestException($"Fetch failed with status {(int)response.StatusCode} {response.StatusCode} from {request.RequestUri}. Body preview: {body}");
|
||||
}
|
||||
|
||||
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
var headers = CreateHeaderDictionary(response);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength ?? contentBytes.LongLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Ok);
|
||||
return SourceFetchContentResult.Success(
|
||||
response.StatusCode,
|
||||
contentBytes,
|
||||
response.Headers.ETag?.Tag,
|
||||
response.Content.Headers.LastModified,
|
||||
response.Content.Headers.ContentType?.ToString(),
|
||||
sendResult.Attempts,
|
||||
headers);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<SourceFetchSendResult> SendAsync(SourceFetchRequest request, HttpCompletionOption completionOption, CancellationToken cancellationToken)
|
||||
{
|
||||
var attemptCount = 0;
|
||||
var options = _httpClientOptions.Get(request.ClientName);
|
||||
|
||||
var response = await SourceRetryPolicy.SendWithRetryAsync(
|
||||
() => CreateHttpRequestMessage(request),
|
||||
async (httpRequest, ct) =>
|
||||
{
|
||||
attemptCount++;
|
||||
var client = _httpClientFactory.CreateClient(request.ClientName);
|
||||
if (request.TimeoutOverride.HasValue)
|
||||
{
|
||||
client.Timeout = request.TimeoutOverride.Value;
|
||||
}
|
||||
|
||||
return await client.SendAsync(httpRequest, completionOption, ct).ConfigureAwait(false);
|
||||
},
|
||||
maxAttempts: options.MaxAttempts,
|
||||
baseDelay: options.BaseDelay,
|
||||
_jitterSource,
|
||||
context => SourceDiagnostics.RecordRetry(
|
||||
request.SourceName,
|
||||
request.ClientName,
|
||||
context.Response?.StatusCode,
|
||||
context.Attempt,
|
||||
context.Delay),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new SourceFetchSendResult(response, attemptCount);
|
||||
}
|
||||
|
||||
internal static HttpRequestMessage CreateHttpRequestMessage(SourceFetchRequest request)
|
||||
{
|
||||
var httpRequest = new HttpRequestMessage(request.Method, request.RequestUri);
|
||||
var acceptValues = request.AcceptHeaders is { Count: > 0 } headers
|
||||
? headers
|
||||
: DefaultAcceptHeaders;
|
||||
|
||||
httpRequest.Headers.Accept.Clear();
|
||||
var added = false;
|
||||
foreach (var mediaType in acceptValues)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(mediaType))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (MediaTypeWithQualityHeaderValue.TryParse(mediaType, out var headerValue))
|
||||
{
|
||||
httpRequest.Headers.Accept.Add(headerValue);
|
||||
added = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!added)
|
||||
{
|
||||
httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultAcceptHeaders[0]));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(request.ETag))
|
||||
{
|
||||
if (System.Net.Http.Headers.EntityTagHeaderValue.TryParse(request.ETag, out var etag))
|
||||
{
|
||||
httpRequest.Headers.IfNoneMatch.Add(etag);
|
||||
}
|
||||
}
|
||||
|
||||
if (request.LastModified.HasValue)
|
||||
{
|
||||
httpRequest.Headers.IfModifiedSince = request.LastModified.Value;
|
||||
}
|
||||
|
||||
return httpRequest;
|
||||
}
|
||||
|
||||
private static async Task<string> ReadResponsePreviewAsync(HttpResponseMessage response, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var buffer = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
var preview = Encoding.UTF8.GetString(buffer);
|
||||
return preview.Length > 256 ? preview[..256] : preview;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return "<unavailable>";
|
||||
}
|
||||
}
|
||||
|
||||
private static string? TryGetHeaderValue(HttpResponseHeaders headers, string name)
|
||||
{
|
||||
if (headers.TryGetValues(name, out var values))
|
||||
{
|
||||
return values.FirstOrDefault();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> CreateHeaderDictionary(HttpResponseMessage response)
|
||||
{
|
||||
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
headers[header.Key] = string.Join(",", header.Value);
|
||||
}
|
||||
|
||||
foreach (var header in response.Content.Headers)
|
||||
{
|
||||
headers[header.Key] = string.Join(",", header.Value);
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private readonly record struct SourceFetchSendResult(HttpResponseMessage Response, int Attempts);
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.Fetch;
|
||||
|
||||
/// <summary>
|
||||
/// Provides retry/backoff behavior for source HTTP fetches.
|
||||
/// </summary>
|
||||
internal static class SourceRetryPolicy
|
||||
{
|
||||
private static readonly StringComparer HeaderComparer = StringComparer.OrdinalIgnoreCase;
|
||||
|
||||
public static async Task<HttpResponseMessage> SendWithRetryAsync(
|
||||
Func<HttpRequestMessage> requestFactory,
|
||||
Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> sender,
|
||||
int maxAttempts,
|
||||
TimeSpan baseDelay,
|
||||
IJitterSource jitterSource,
|
||||
Action<SourceRetryAttemptContext>? onRetry,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(requestFactory);
|
||||
ArgumentNullException.ThrowIfNull(sender);
|
||||
ArgumentNullException.ThrowIfNull(jitterSource);
|
||||
|
||||
var attempt = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
attempt++;
|
||||
using var request = requestFactory();
|
||||
HttpResponseMessage response;
|
||||
|
||||
try
|
||||
{
|
||||
response = await sender(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (attempt < maxAttempts)
|
||||
{
|
||||
var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource);
|
||||
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay));
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (NeedsRetry(response) && attempt < maxAttempts)
|
||||
{
|
||||
var delay = ComputeDelay(
|
||||
baseDelay,
|
||||
attempt,
|
||||
GetRetryAfter(response),
|
||||
jitterSource);
|
||||
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, response, null, delay));
|
||||
response.Dispose();
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool NeedsRetry(HttpResponseMessage response)
|
||||
{
|
||||
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (IsRateLimitResponse(response))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var status = (int)response.StatusCode;
|
||||
return status >= 500 && status < 600;
|
||||
}
|
||||
|
||||
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
|
||||
{
|
||||
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
|
||||
{
|
||||
return retryAfter.Value;
|
||||
}
|
||||
|
||||
var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1));
|
||||
var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250))
|
||||
?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250));
|
||||
return exponential + jitter;
|
||||
}
|
||||
|
||||
private static bool IsRateLimitResponse(HttpResponseMessage response)
|
||||
{
|
||||
if (response.Headers.RetryAfter is not null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (response.StatusCode == System.Net.HttpStatusCode.Forbidden || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
if (TryGetRateLimitRemaining(response, out var remaining) && remaining <= 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (response.Headers.TryGetValues("X-RateLimit-Reset", out var _))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryGetRateLimitRemaining(HttpResponseMessage response, out long remaining)
|
||||
{
|
||||
remaining = 0;
|
||||
if (response.Headers.TryGetValues("X-RateLimit-Remaining", out var values))
|
||||
{
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
remaining = parsed;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static TimeSpan? GetRetryAfter(HttpResponseMessage response)
|
||||
{
|
||||
var retryAfter = response.Headers.RetryAfter;
|
||||
if (retryAfter is not null)
|
||||
{
|
||||
if (retryAfter.Delta.HasValue && retryAfter.Delta.Value > TimeSpan.Zero)
|
||||
{
|
||||
return retryAfter.Delta;
|
||||
}
|
||||
|
||||
if (retryAfter.Date.HasValue)
|
||||
{
|
||||
var delta = retryAfter.Date.Value - DateTimeOffset.UtcNow;
|
||||
if (delta > TimeSpan.Zero)
|
||||
{
|
||||
return delta;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (response.Headers.TryGetValues("Retry-After", out var retryAfterValues))
|
||||
{
|
||||
foreach (var value in retryAfterValues)
|
||||
{
|
||||
if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) && seconds > 0)
|
||||
{
|
||||
return TimeSpan.FromSeconds(seconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (response.Headers.TryGetValues("X-RateLimit-Reset", out var resetValues))
|
||||
{
|
||||
foreach (var value in resetValues)
|
||||
{
|
||||
if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var epochSeconds))
|
||||
{
|
||||
var resetTime = DateTimeOffset.FromUnixTimeSeconds(epochSeconds);
|
||||
var delta = resetTime - DateTimeOffset.UtcNow;
|
||||
if (delta > TimeSpan.Zero)
|
||||
{
|
||||
return delta;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
internal readonly record struct SourceRetryAttemptContext(int Attempt, HttpResponseMessage? Response, Exception? Exception, TimeSpan Delay);
|
||||
Reference in New Issue
Block a user