up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,97 +1,97 @@
namespace StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
public sealed class RedHatOptions
{
/// <summary>
/// Name of the HttpClient registered for Red Hat Hydra requests.
/// </summary>
public const string HttpClientName = "redhat-hydra";
/// <summary>
/// Base API endpoint for Hydra security data requests.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://access.redhat.com/hydra/rest/securitydata");
/// <summary>
/// Relative path for the advisory listing endpoint (returns summary rows with resource_url values).
/// </summary>
public string SummaryPath { get; set; } = "csaf.json";
/// <summary>
/// Number of summary rows requested per page when scanning for new advisories.
/// </summary>
public int PageSize { get; set; } = 200;
/// <summary>
/// Maximum number of summary pages to inspect within one fetch invocation.
/// </summary>
public int MaxPagesPerFetch { get; set; } = 5;
/// <summary>
/// Upper bound on individual advisories fetched per invocation (guards against unbounded catch-up floods).
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 800;
/// <summary>
/// Initial look-back window applied when no watermark exists (Red Hat publishes extensive history; we default to 30 days).
/// </summary>
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
/// <summary>
/// Optional overlap period re-scanned on each run to pick up late-published advisories.
/// </summary>
public TimeSpan Overlap { get; set; } = TimeSpan.FromDays(1);
/// <summary>
/// Timeout applied to individual Hydra document fetches.
/// </summary>
public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(60);
/// <summary>
/// Custom user-agent presented to Red Hat endpoints (kept short to satisfy Jetty header limits).
/// </summary>
public string UserAgent { get; set; } = "StellaOps.Concelier.RedHat/1.0";
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("Red Hat Hydra base endpoint must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(SummaryPath))
{
throw new InvalidOperationException("Red Hat Hydra summary path must be configured.");
}
if (PageSize <= 0)
{
throw new InvalidOperationException("Red Hat Hydra page size must be positive.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("Red Hat Hydra max pages per fetch must be positive.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException("Red Hat Hydra max advisories per fetch must be positive.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Red Hat Hydra initial backfill must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Red Hat Hydra overlap cannot be negative.");
}
if (FetchTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("Red Hat Hydra fetch timeout must be positive.");
}
}
}
namespace StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
public sealed class RedHatOptions
{
/// <summary>
/// Name of the HttpClient registered for Red Hat Hydra requests.
/// </summary>
public const string HttpClientName = "redhat-hydra";
/// <summary>
/// Base API endpoint for Hydra security data requests.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://access.redhat.com/hydra/rest/securitydata");
/// <summary>
/// Relative path for the advisory listing endpoint (returns summary rows with resource_url values).
/// </summary>
public string SummaryPath { get; set; } = "csaf.json";
/// <summary>
/// Number of summary rows requested per page when scanning for new advisories.
/// </summary>
public int PageSize { get; set; } = 200;
/// <summary>
/// Maximum number of summary pages to inspect within one fetch invocation.
/// </summary>
public int MaxPagesPerFetch { get; set; } = 5;
/// <summary>
/// Upper bound on individual advisories fetched per invocation (guards against unbounded catch-up floods).
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 800;
/// <summary>
/// Initial look-back window applied when no watermark exists (Red Hat publishes extensive history; we default to 30 days).
/// </summary>
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
/// <summary>
/// Optional overlap period re-scanned on each run to pick up late-published advisories.
/// </summary>
public TimeSpan Overlap { get; set; } = TimeSpan.FromDays(1);
/// <summary>
/// Timeout applied to individual Hydra document fetches.
/// </summary>
public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(60);
/// <summary>
/// Custom user-agent presented to Red Hat endpoints (kept short to satisfy Jetty header limits).
/// </summary>
public string UserAgent { get; set; } = "StellaOps.Concelier.RedHat/1.0";
public void Validate()
{
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("Red Hat Hydra base endpoint must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(SummaryPath))
{
throw new InvalidOperationException("Red Hat Hydra summary path must be configured.");
}
if (PageSize <= 0)
{
throw new InvalidOperationException("Red Hat Hydra page size must be positive.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("Red Hat Hydra max pages per fetch must be positive.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException("Red Hat Hydra max advisories per fetch must be positive.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Red Hat Hydra initial backfill must be positive.");
}
if (Overlap < TimeSpan.Zero)
{
throw new InvalidOperationException("Red Hat Hydra overlap cannot be negative.");
}
if (FetchTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("Red Hat Hydra fetch timeout must be positive.");
}
}
}

View File

@@ -1,177 +1,177 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal.Models;
internal sealed class RedHatCsafEnvelope
{
[JsonPropertyName("document")]
public RedHatDocumentSection? Document { get; init; }
[JsonPropertyName("product_tree")]
public RedHatProductTree? ProductTree { get; init; }
[JsonPropertyName("vulnerabilities")]
public IReadOnlyList<RedHatVulnerability>? Vulnerabilities { get; init; }
}
internal sealed class RedHatDocumentSection
{
[JsonPropertyName("aggregate_severity")]
public RedHatAggregateSeverity? AggregateSeverity { get; init; }
[JsonPropertyName("lang")]
public string? Lang { get; init; }
[JsonPropertyName("notes")]
public IReadOnlyList<RedHatDocumentNote>? Notes { get; init; }
[JsonPropertyName("references")]
public IReadOnlyList<RedHatReference>? References { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("tracking")]
public RedHatTracking? Tracking { get; init; }
}
internal sealed class RedHatAggregateSeverity
{
[JsonPropertyName("text")]
public string? Text { get; init; }
}
internal sealed class RedHatDocumentNote
{
[JsonPropertyName("category")]
public string? Category { get; init; }
[JsonPropertyName("text")]
public string? Text { get; init; }
public bool CategoryEquals(string value)
=> !string.IsNullOrWhiteSpace(Category)
&& string.Equals(Category, value, StringComparison.OrdinalIgnoreCase);
}
internal sealed class RedHatTracking
{
[JsonPropertyName("id")]
public string? Id { get; init; }
[JsonPropertyName("initial_release_date")]
public DateTimeOffset? InitialReleaseDate { get; init; }
[JsonPropertyName("current_release_date")]
public DateTimeOffset? CurrentReleaseDate { get; init; }
}
internal sealed class RedHatReference
{
[JsonPropertyName("category")]
public string? Category { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("url")]
public string? Url { get; init; }
}
internal sealed class RedHatProductTree
{
[JsonPropertyName("branches")]
public IReadOnlyList<RedHatProductBranch>? Branches { get; init; }
}
internal sealed class RedHatProductBranch
{
[JsonPropertyName("category")]
public string? Category { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("product")]
public RedHatProductNodeInfo? Product { get; init; }
[JsonPropertyName("branches")]
public IReadOnlyList<RedHatProductBranch>? Branches { get; init; }
}
internal sealed class RedHatProductNodeInfo
{
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("product_id")]
public string? ProductId { get; init; }
[JsonPropertyName("product_identification_helper")]
public RedHatProductIdentificationHelper? ProductIdentificationHelper { get; init; }
}
internal sealed class RedHatProductIdentificationHelper
{
[JsonPropertyName("cpe")]
public string? Cpe { get; init; }
[JsonPropertyName("purl")]
public string? Purl { get; init; }
}
internal sealed class RedHatVulnerability
{
[JsonPropertyName("cve")]
public string? Cve { get; init; }
[JsonPropertyName("references")]
public IReadOnlyList<RedHatReference>? References { get; init; }
[JsonPropertyName("scores")]
public IReadOnlyList<RedHatVulnerabilityScore>? Scores { get; init; }
[JsonPropertyName("product_status")]
public RedHatProductStatus? ProductStatus { get; init; }
}
internal sealed class RedHatVulnerabilityScore
{
[JsonPropertyName("cvss_v3")]
public RedHatCvssV3? CvssV3 { get; init; }
}
internal sealed class RedHatCvssV3
{
[JsonPropertyName("baseScore")]
public double? BaseScore { get; init; }
[JsonPropertyName("baseSeverity")]
public string? BaseSeverity { get; init; }
[JsonPropertyName("vectorString")]
public string? VectorString { get; init; }
[JsonPropertyName("version")]
public string? Version { get; init; }
}
internal sealed class RedHatProductStatus
{
[JsonPropertyName("fixed")]
public IReadOnlyList<string>? Fixed { get; init; }
[JsonPropertyName("first_fixed")]
public IReadOnlyList<string>? FirstFixed { get; init; }
[JsonPropertyName("known_affected")]
public IReadOnlyList<string>? KnownAffected { get; init; }
[JsonPropertyName("known_not_affected")]
public IReadOnlyList<string>? KnownNotAffected { get; init; }
[JsonPropertyName("under_investigation")]
public IReadOnlyList<string>? UnderInvestigation { get; init; }
}
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal.Models;
internal sealed class RedHatCsafEnvelope
{
[JsonPropertyName("document")]
public RedHatDocumentSection? Document { get; init; }
[JsonPropertyName("product_tree")]
public RedHatProductTree? ProductTree { get; init; }
[JsonPropertyName("vulnerabilities")]
public IReadOnlyList<RedHatVulnerability>? Vulnerabilities { get; init; }
}
internal sealed class RedHatDocumentSection
{
[JsonPropertyName("aggregate_severity")]
public RedHatAggregateSeverity? AggregateSeverity { get; init; }
[JsonPropertyName("lang")]
public string? Lang { get; init; }
[JsonPropertyName("notes")]
public IReadOnlyList<RedHatDocumentNote>? Notes { get; init; }
[JsonPropertyName("references")]
public IReadOnlyList<RedHatReference>? References { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("tracking")]
public RedHatTracking? Tracking { get; init; }
}
internal sealed class RedHatAggregateSeverity
{
[JsonPropertyName("text")]
public string? Text { get; init; }
}
internal sealed class RedHatDocumentNote
{
[JsonPropertyName("category")]
public string? Category { get; init; }
[JsonPropertyName("text")]
public string? Text { get; init; }
public bool CategoryEquals(string value)
=> !string.IsNullOrWhiteSpace(Category)
&& string.Equals(Category, value, StringComparison.OrdinalIgnoreCase);
}
internal sealed class RedHatTracking
{
[JsonPropertyName("id")]
public string? Id { get; init; }
[JsonPropertyName("initial_release_date")]
public DateTimeOffset? InitialReleaseDate { get; init; }
[JsonPropertyName("current_release_date")]
public DateTimeOffset? CurrentReleaseDate { get; init; }
}
internal sealed class RedHatReference
{
[JsonPropertyName("category")]
public string? Category { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("url")]
public string? Url { get; init; }
}
internal sealed class RedHatProductTree
{
[JsonPropertyName("branches")]
public IReadOnlyList<RedHatProductBranch>? Branches { get; init; }
}
internal sealed class RedHatProductBranch
{
[JsonPropertyName("category")]
public string? Category { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("product")]
public RedHatProductNodeInfo? Product { get; init; }
[JsonPropertyName("branches")]
public IReadOnlyList<RedHatProductBranch>? Branches { get; init; }
}
internal sealed class RedHatProductNodeInfo
{
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("product_id")]
public string? ProductId { get; init; }
[JsonPropertyName("product_identification_helper")]
public RedHatProductIdentificationHelper? ProductIdentificationHelper { get; init; }
}
internal sealed class RedHatProductIdentificationHelper
{
[JsonPropertyName("cpe")]
public string? Cpe { get; init; }
[JsonPropertyName("purl")]
public string? Purl { get; init; }
}
internal sealed class RedHatVulnerability
{
[JsonPropertyName("cve")]
public string? Cve { get; init; }
[JsonPropertyName("references")]
public IReadOnlyList<RedHatReference>? References { get; init; }
[JsonPropertyName("scores")]
public IReadOnlyList<RedHatVulnerabilityScore>? Scores { get; init; }
[JsonPropertyName("product_status")]
public RedHatProductStatus? ProductStatus { get; init; }
}
internal sealed class RedHatVulnerabilityScore
{
[JsonPropertyName("cvss_v3")]
public RedHatCvssV3? CvssV3 { get; init; }
}
internal sealed class RedHatCvssV3
{
[JsonPropertyName("baseScore")]
public double? BaseScore { get; init; }
[JsonPropertyName("baseSeverity")]
public string? BaseSeverity { get; init; }
[JsonPropertyName("vectorString")]
public string? VectorString { get; init; }
[JsonPropertyName("version")]
public string? Version { get; init; }
}
internal sealed class RedHatProductStatus
{
[JsonPropertyName("fixed")]
public IReadOnlyList<string>? Fixed { get; init; }
[JsonPropertyName("first_fixed")]
public IReadOnlyList<string>? FirstFixed { get; init; }
[JsonPropertyName("known_affected")]
public IReadOnlyList<string>? KnownAffected { get; init; }
[JsonPropertyName("known_not_affected")]
public IReadOnlyList<string>? KnownNotAffected { get; init; }
[JsonPropertyName("under_investigation")]
public IReadOnlyList<string>? UnderInvestigation { get; init; }
}

View File

@@ -1,254 +1,254 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal;
internal sealed record RedHatCursor(
DateTimeOffset? LastReleasedOn,
IReadOnlyCollection<string> ProcessedAdvisoryIds,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, RedHatCachedFetchMetadata> FetchCache)
{
private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>();
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, RedHatCachedFetchMetadata> EmptyCache =
new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase);
public static RedHatCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyCache);
public static RedHatCursor FromBsonDocument(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
DateTimeOffset? lastReleased = null;
if (document.TryGetValue("lastReleasedOn", out var lastReleasedValue))
{
lastReleased = ReadDateTimeOffset(lastReleasedValue);
}
var processed = ReadStringSet(document, "processedAdvisories");
var pendingDocuments = ReadGuidSet(document, "pendingDocuments");
var pendingMappings = ReadGuidSet(document, "pendingMappings");
var fetchCache = ReadFetchCache(document);
return new RedHatCursor(lastReleased, processed, pendingDocuments, pendingMappings, fetchCache);
}
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument();
if (LastReleasedOn.HasValue)
{
document["lastReleasedOn"] = LastReleasedOn.Value.UtcDateTime;
}
document["processedAdvisories"] = new BsonArray(ProcessedAdvisoryIds);
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString()));
document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString()));
var cacheArray = new BsonArray();
foreach (var (key, metadata) in FetchCache)
{
var cacheDoc = new BsonDocument
{
["uri"] = key
};
if (!string.IsNullOrWhiteSpace(metadata.ETag))
{
cacheDoc["etag"] = metadata.ETag;
}
if (metadata.LastModified.HasValue)
{
cacheDoc["lastModified"] = metadata.LastModified.Value.UtcDateTime;
}
cacheArray.Add(cacheDoc);
}
document["fetchCache"] = cacheArray;
return document;
}
public RedHatCursor WithLastReleased(DateTimeOffset? releasedOn, IEnumerable<string> advisoryIds)
{
var normalizedIds = advisoryIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>();
return this with
{
LastReleasedOn = releasedOn,
ProcessedAdvisoryIds = normalizedIds
};
}
public RedHatCursor AddProcessedAdvisories(IEnumerable<string> advisoryIds)
{
if (advisoryIds is null)
{
return this;
}
var set = new HashSet<string>(ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase);
foreach (var id in advisoryIds)
{
if (!string.IsNullOrWhiteSpace(id))
{
set.Add(id.Trim());
}
}
return this with { ProcessedAdvisoryIds = set.ToArray() };
}
public RedHatCursor WithPendingDocuments(IEnumerable<Guid> ids)
{
var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>();
return this with { PendingDocuments = list };
}
public RedHatCursor WithPendingMappings(IEnumerable<Guid> ids)
{
var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>();
return this with { PendingMappings = list };
}
public RedHatCursor WithFetchCache(string requestUri, string? etag, DateTimeOffset? lastModified)
{
var cache = new Dictionary<string, RedHatCachedFetchMetadata>(FetchCache, StringComparer.OrdinalIgnoreCase)
{
[requestUri] = new RedHatCachedFetchMetadata(etag, lastModified)
};
return this with { FetchCache = cache };
}
public RedHatCursor PruneFetchCache(IEnumerable<string> keepUris)
{
if (FetchCache.Count == 0)
{
return this;
}
var keepSet = new HashSet<string>(keepUris ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
if (keepSet.Count == 0)
{
return this with { FetchCache = EmptyCache };
}
var cache = new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var uri in keepSet)
{
if (FetchCache.TryGetValue(uri, out var metadata))
{
cache[uri] = metadata;
}
}
return this with { FetchCache = cache };
}
public RedHatCachedFetchMetadata? TryGetFetchCache(string requestUri)
{
if (FetchCache.TryGetValue(requestUri, out var metadata))
{
return metadata;
}
return null;
}
private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyStringList;
}
var results = new List<string>(array.Count);
foreach (var element in array)
{
if (element.BsonType == BsonType.String)
{
var str = element.AsString.Trim();
if (!string.IsNullOrWhiteSpace(str))
{
results.Add(str);
}
}
}
return results;
}
private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidList;
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element.BsonType == BsonType.String && Guid.TryParse(element.AsString, out var guid))
{
results.Add(guid);
}
}
return results;
}
private static IReadOnlyDictionary<string, RedHatCachedFetchMetadata> ReadFetchCache(BsonDocument document)
{
if (!document.TryGetValue("fetchCache", out var value) || value is not BsonArray array || array.Count == 0)
{
return EmptyCache;
}
var results = new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var element in array.OfType<BsonDocument>())
{
if (!element.TryGetValue("uri", out var uriValue) || uriValue.BsonType != BsonType.String)
{
continue;
}
var uri = uriValue.AsString;
var etag = element.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String
? etagValue.AsString
: null;
DateTimeOffset? lastModified = null;
if (element.TryGetValue("lastModified", out var lastModifiedValue))
{
lastModified = ReadDateTimeOffset(lastModifiedValue);
}
results[uri] = new RedHatCachedFetchMetadata(etag, lastModified);
}
return results;
}
private static DateTimeOffset? ReadDateTimeOffset(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
internal sealed record RedHatCachedFetchMetadata(string? ETag, DateTimeOffset? LastModified);
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal;
internal sealed record RedHatCursor(
DateTimeOffset? LastReleasedOn,
IReadOnlyCollection<string> ProcessedAdvisoryIds,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, RedHatCachedFetchMetadata> FetchCache)
{
private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>();
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, RedHatCachedFetchMetadata> EmptyCache =
new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase);
public static RedHatCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyCache);
public static RedHatCursor FromDocumentObject(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
DateTimeOffset? lastReleased = null;
if (document.TryGetValue("lastReleasedOn", out var lastReleasedValue))
{
lastReleased = ReadDateTimeOffset(lastReleasedValue);
}
var processed = ReadStringSet(document, "processedAdvisories");
var pendingDocuments = ReadGuidSet(document, "pendingDocuments");
var pendingMappings = ReadGuidSet(document, "pendingMappings");
var fetchCache = ReadFetchCache(document);
return new RedHatCursor(lastReleased, processed, pendingDocuments, pendingMappings, fetchCache);
}
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject();
if (LastReleasedOn.HasValue)
{
document["lastReleasedOn"] = LastReleasedOn.Value.UtcDateTime;
}
document["processedAdvisories"] = new DocumentArray(ProcessedAdvisoryIds);
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString()));
var cacheArray = new DocumentArray();
foreach (var (key, metadata) in FetchCache)
{
var cacheDoc = new DocumentObject
{
["uri"] = key
};
if (!string.IsNullOrWhiteSpace(metadata.ETag))
{
cacheDoc["etag"] = metadata.ETag;
}
if (metadata.LastModified.HasValue)
{
cacheDoc["lastModified"] = metadata.LastModified.Value.UtcDateTime;
}
cacheArray.Add(cacheDoc);
}
document["fetchCache"] = cacheArray;
return document;
}
public RedHatCursor WithLastReleased(DateTimeOffset? releasedOn, IEnumerable<string> advisoryIds)
{
var normalizedIds = advisoryIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>();
return this with
{
LastReleasedOn = releasedOn,
ProcessedAdvisoryIds = normalizedIds
};
}
public RedHatCursor AddProcessedAdvisories(IEnumerable<string> advisoryIds)
{
if (advisoryIds is null)
{
return this;
}
var set = new HashSet<string>(ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase);
foreach (var id in advisoryIds)
{
if (!string.IsNullOrWhiteSpace(id))
{
set.Add(id.Trim());
}
}
return this with { ProcessedAdvisoryIds = set.ToArray() };
}
public RedHatCursor WithPendingDocuments(IEnumerable<Guid> ids)
{
var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>();
return this with { PendingDocuments = list };
}
public RedHatCursor WithPendingMappings(IEnumerable<Guid> ids)
{
var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>();
return this with { PendingMappings = list };
}
public RedHatCursor WithFetchCache(string requestUri, string? etag, DateTimeOffset? lastModified)
{
var cache = new Dictionary<string, RedHatCachedFetchMetadata>(FetchCache, StringComparer.OrdinalIgnoreCase)
{
[requestUri] = new RedHatCachedFetchMetadata(etag, lastModified)
};
return this with { FetchCache = cache };
}
public RedHatCursor PruneFetchCache(IEnumerable<string> keepUris)
{
if (FetchCache.Count == 0)
{
return this;
}
var keepSet = new HashSet<string>(keepUris ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
if (keepSet.Count == 0)
{
return this with { FetchCache = EmptyCache };
}
var cache = new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var uri in keepSet)
{
if (FetchCache.TryGetValue(uri, out var metadata))
{
cache[uri] = metadata;
}
}
return this with { FetchCache = cache };
}
public RedHatCachedFetchMetadata? TryGetFetchCache(string requestUri)
{
if (FetchCache.TryGetValue(requestUri, out var metadata))
{
return metadata;
}
return null;
}
private static IReadOnlyCollection<string> ReadStringSet(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyStringList;
}
var results = new List<string>(array.Count);
foreach (var element in array)
{
if (element.DocumentType == DocumentType.String)
{
var str = element.AsString.Trim();
if (!string.IsNullOrWhiteSpace(str))
{
results.Add(str);
}
}
}
return results;
}
private static IReadOnlyCollection<Guid> ReadGuidSet(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidList;
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element.DocumentType == DocumentType.String && Guid.TryParse(element.AsString, out var guid))
{
results.Add(guid);
}
}
return results;
}
private static IReadOnlyDictionary<string, RedHatCachedFetchMetadata> ReadFetchCache(DocumentObject document)
{
if (!document.TryGetValue("fetchCache", out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyCache;
}
var results = new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var element in array.OfType<DocumentObject>())
{
if (!element.TryGetValue("uri", out var uriValue) || uriValue.DocumentType != DocumentType.String)
{
continue;
}
var uri = uriValue.AsString;
var etag = element.TryGetValue("etag", out var etagValue) && etagValue.DocumentType == DocumentType.String
? etagValue.AsString
: null;
DateTimeOffset? lastModified = null;
if (element.TryGetValue("lastModified", out var lastModifiedValue))
{
lastModified = ReadDateTimeOffset(lastModifiedValue);
}
results[uri] = new RedHatCachedFetchMetadata(etag, lastModified);
}
return results;
}
private static DateTimeOffset? ReadDateTimeOffset(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
internal sealed record RedHatCachedFetchMetadata(string? ETag, DateTimeOffset? LastModified);

View File

@@ -1,66 +1,66 @@
using System;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal;
internal readonly record struct RedHatSummaryItem(string AdvisoryId, DateTimeOffset ReleasedOn, Uri ResourceUri)
{
private static readonly string[] AdvisoryFields =
{
"RHSA",
"RHBA",
"RHEA",
"RHUI",
"RHBG",
"RHBO",
"advisory"
};
public static bool TryParse(JsonElement element, out RedHatSummaryItem item)
{
item = default;
string? advisoryId = null;
foreach (var field in AdvisoryFields)
{
if (element.TryGetProperty(field, out var advisoryProperty) && advisoryProperty.ValueKind == JsonValueKind.String)
{
var candidate = advisoryProperty.GetString();
if (!string.IsNullOrWhiteSpace(candidate))
{
advisoryId = candidate.Trim();
break;
}
}
}
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
if (!element.TryGetProperty("released_on", out var releasedProperty) || releasedProperty.ValueKind != JsonValueKind.String)
{
return false;
}
if (!DateTimeOffset.TryParse(releasedProperty.GetString(), out var releasedOn))
{
return false;
}
if (!element.TryGetProperty("resource_url", out var resourceProperty) || resourceProperty.ValueKind != JsonValueKind.String)
{
return false;
}
var resourceValue = resourceProperty.GetString();
if (!Uri.TryCreate(resourceValue, UriKind.Absolute, out var resourceUri))
{
return false;
}
item = new RedHatSummaryItem(advisoryId!, releasedOn.ToUniversalTime(), resourceUri);
return true;
}
}
using System;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal;
internal readonly record struct RedHatSummaryItem(string AdvisoryId, DateTimeOffset ReleasedOn, Uri ResourceUri)
{
private static readonly string[] AdvisoryFields =
{
"RHSA",
"RHBA",
"RHEA",
"RHUI",
"RHBG",
"RHBO",
"advisory"
};
public static bool TryParse(JsonElement element, out RedHatSummaryItem item)
{
item = default;
string? advisoryId = null;
foreach (var field in AdvisoryFields)
{
if (element.TryGetProperty(field, out var advisoryProperty) && advisoryProperty.ValueKind == JsonValueKind.String)
{
var candidate = advisoryProperty.GetString();
if (!string.IsNullOrWhiteSpace(candidate))
{
advisoryId = candidate.Trim();
break;
}
}
}
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
if (!element.TryGetProperty("released_on", out var releasedProperty) || releasedProperty.ValueKind != JsonValueKind.String)
{
return false;
}
if (!DateTimeOffset.TryParse(releasedProperty.GetString(), out var releasedOn))
{
return false;
}
if (!element.TryGetProperty("resource_url", out var resourceProperty) || resourceProperty.ValueKind != JsonValueKind.String)
{
return false;
}
var resourceValue = resourceProperty.GetString();
if (!Uri.TryCreate(resourceValue, UriKind.Absolute, out var resourceUri))
{
return false;
}
item = new RedHatSummaryItem(advisoryId!, releasedOn.ToUniversalTime(), resourceUri);
return true;
}
}

View File

@@ -1,46 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
internal static class RedHatJobKinds
{
public const string Fetch = "source:redhat:fetch";
public const string Parse = "source:redhat:parse";
public const string Map = "source:redhat:map";
}
internal sealed class RedHatFetchJob : IJob
{
private readonly RedHatConnector _connector;
public RedHatFetchJob(RedHatConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class RedHatParseJob : IJob
{
private readonly RedHatConnector _connector;
public RedHatParseJob(RedHatConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class RedHatMapJob : IJob
{
private readonly RedHatConnector _connector;
public RedHatMapJob(RedHatConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
internal static class RedHatJobKinds
{
public const string Fetch = "source:redhat:fetch";
public const string Parse = "source:redhat:parse";
public const string Map = "source:redhat:map";
}
internal sealed class RedHatFetchJob : IJob
{
private readonly RedHatConnector _connector;
public RedHatFetchJob(RedHatConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class RedHatParseJob : IJob
{
private readonly RedHatConnector _connector;
public RedHatParseJob(RedHatConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class RedHatMapJob : IJob
{
private readonly RedHatConnector _connector;
public RedHatMapJob(RedHatConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Distro.RedHat.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Distro.RedHat.Tests")]

View File

@@ -5,8 +5,8 @@ using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Bson.IO;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Documents.IO;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
@@ -312,7 +312,7 @@ public sealed class RedHatConnector : IFeedConnector
var rawBytes = await _rawDocumentStorage.DownloadAsync(document.PayloadId.Value, cancellationToken).ConfigureAwait(false);
using var jsonDocument = JsonDocument.Parse(rawBytes);
var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement);
var payload = BsonDocument.Parse(sanitized);
var payload = DocumentObject.Parse(sanitized);
var dtoRecord = new DtoRecord(
Guid.NewGuid(),
@@ -402,13 +402,13 @@ public sealed class RedHatConnector : IFeedConnector
private async Task<RedHatCursor> GetCursorAsync(CancellationToken cancellationToken)
{
var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
return RedHatCursor.FromBsonDocument(record?.Cursor);
return RedHatCursor.FromDocumentObject(record?.Cursor);
}
private async Task UpdateCursorAsync(RedHatCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
private Uri BuildSummaryUri(DateTimeOffset after, int page)

View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
public sealed class RedHatConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "redhat";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<RedHatConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
public sealed class RedHatConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "redhat";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<RedHatConnector>(services);
}
}

View File

@@ -1,54 +1,54 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
public sealed class RedHatDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:redhat";
private const string FetchCron = "0,15,30,45 * * * *";
private const string ParseCron = "5,20,35,50 * * * *";
private const string MapCron = "10,25,40,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(12);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(15);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(6);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddRedHatConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var schedulerBuilder = new JobSchedulerBuilder(services);
schedulerBuilder
.AddJob<RedHatFetchJob>(
RedHatJobKinds.Fetch,
cronExpression: FetchCron,
timeout: FetchTimeout,
leaseDuration: LeaseDuration)
.AddJob<RedHatParseJob>(
RedHatJobKinds.Parse,
cronExpression: ParseCron,
timeout: ParseTimeout,
leaseDuration: LeaseDuration)
.AddJob<RedHatMapJob>(
RedHatJobKinds.Map,
cronExpression: MapCron,
timeout: MapTimeout,
leaseDuration: LeaseDuration);
return services;
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
public sealed class RedHatDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:redhat";
private const string FetchCron = "0,15,30,45 * * * *";
private const string ParseCron = "5,20,35,50 * * * *";
private const string MapCron = "10,25,40,55 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(12);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(15);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(6);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddRedHatConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var schedulerBuilder = new JobSchedulerBuilder(services);
schedulerBuilder
.AddJob<RedHatFetchJob>(
RedHatJobKinds.Fetch,
cronExpression: FetchCron,
timeout: FetchTimeout,
leaseDuration: LeaseDuration)
.AddJob<RedHatParseJob>(
RedHatJobKinds.Parse,
cronExpression: ParseCron,
timeout: ParseTimeout,
leaseDuration: LeaseDuration)
.AddJob<RedHatMapJob>(
RedHatJobKinds.Map,
cronExpression: MapCron,
timeout: MapTimeout,
leaseDuration: LeaseDuration);
return services;
}
}

View File

@@ -1,34 +1,34 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
public static class RedHatServiceCollectionExtensions
{
public static IServiceCollection AddRedHatConnector(this IServiceCollection services, Action<RedHatOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<RedHatOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(RedHatOptions.HttpClientName, (sp, httpOptions) =>
{
var options = sp.GetRequiredService<IOptions<RedHatOptions>>().Value;
httpOptions.BaseAddress = options.BaseEndpoint;
httpOptions.Timeout = options.FetchTimeout;
httpOptions.UserAgent = options.UserAgent;
httpOptions.AllowedHosts.Clear();
httpOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
httpOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddTransient<RedHatConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
namespace StellaOps.Concelier.Connector.Distro.RedHat;
public static class RedHatServiceCollectionExtensions
{
public static IServiceCollection AddRedHatConnector(this IServiceCollection services, Action<RedHatOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<RedHatOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(RedHatOptions.HttpClientName, (sp, httpOptions) =>
{
var options = sp.GetRequiredService<IOptions<RedHatOptions>>().Value;
httpOptions.BaseAddress = options.BaseEndpoint;
httpOptions.Timeout = options.FetchTimeout;
httpOptions.UserAgent = options.UserAgent;
httpOptions.AllowedHosts.Clear();
httpOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
httpOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddTransient<RedHatConnector>();
return services;
}
}