Rename Concelier Source modules to Connector
This commit is contained in:
master
@@ -0,0 +1,69 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
|
||||
|
||||
public sealed class UbuntuOptions
|
||||
{
|
||||
public const string HttpClientName = "concelier.ubuntu";
|
||||
public const int MaxPageSize = 20;
|
||||
|
||||
/// <summary>
|
||||
/// Endpoint exposing the rolling JSON index of Ubuntu Security Notices.
|
||||
/// </summary>
|
||||
public Uri NoticesEndpoint { get; set; } = new("https://ubuntu.com/security/notices.json");
|
||||
|
||||
/// <summary>
|
||||
/// Base URI where individual notice detail pages live.
|
||||
/// </summary>
|
||||
public Uri NoticeDetailBaseUri { get; set; } = new("https://ubuntu.com/security/");
|
||||
|
||||
public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45);
|
||||
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(3);
|
||||
|
||||
public int MaxNoticesPerFetch { get; set; } = 60;
|
||||
|
||||
public int IndexPageSize { get; set; } = 20;
|
||||
|
||||
public string UserAgent { get; set; } = "StellaOps.Concelier.Ubuntu/0.1 (+https://stella-ops.org)";
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (NoticesEndpoint is null || !NoticesEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Ubuntu notices endpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (NoticeDetailBaseUri is null || !NoticeDetailBaseUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Ubuntu notice detail base URI must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (MaxNoticesPerFetch <= 0 || MaxNoticesPerFetch > 200)
|
||||
{
|
||||
throw new InvalidOperationException("MaxNoticesPerFetch must be between 1 and 200.");
|
||||
}
|
||||
|
||||
if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5))
|
||||
{
|
||||
throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes.");
|
||||
}
|
||||
|
||||
if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365))
|
||||
{
|
||||
throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days.");
|
||||
}
|
||||
|
||||
if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14))
|
||||
{
|
||||
throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days.");
|
||||
}
|
||||
|
||||
if (IndexPageSize <= 0 || IndexPageSize > MaxPageSize)
|
||||
{
|
||||
throw new InvalidOperationException($"IndexPageSize must be between 1 and {MaxPageSize}.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
|
||||
|
||||
internal sealed record UbuntuCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<string> ProcessedNoticeIds,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, UbuntuFetchCacheEntry> FetchCache)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<string> EmptyIds = Array.Empty<string>();
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, UbuntuFetchCacheEntry> EmptyCache =
|
||||
new Dictionary<string, UbuntuFetchCacheEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static UbuntuCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache);
|
||||
|
||||
public static UbuntuCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
DateTimeOffset? lastPublished = null;
|
||||
if (document.TryGetValue("lastPublished", out var value))
|
||||
{
|
||||
lastPublished = value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
var processed = ReadStringSet(document, "processedIds");
|
||||
var pendingDocuments = ReadGuidSet(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidSet(document, "pendingMappings");
|
||||
var cache = ReadCache(document);
|
||||
|
||||
return new UbuntuCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache);
|
||||
}
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var doc = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString()))
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
doc["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (ProcessedNoticeIds.Count > 0)
|
||||
{
|
||||
doc["processedIds"] = new BsonArray(ProcessedNoticeIds);
|
||||
}
|
||||
|
||||
if (FetchCache.Count > 0)
|
||||
{
|
||||
var cacheDoc = new BsonDocument();
|
||||
foreach (var (key, entry) in FetchCache)
|
||||
{
|
||||
cacheDoc[key] = entry.ToBsonDocument();
|
||||
}
|
||||
|
||||
doc["fetchCache"] = cacheDoc;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
public UbuntuCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public UbuntuCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public UbuntuCursor WithFetchCache(IDictionary<string, UbuntuFetchCacheEntry>? cache)
|
||||
{
|
||||
if (cache is null || cache.Count == 0)
|
||||
{
|
||||
return this with { FetchCache = EmptyCache };
|
||||
}
|
||||
|
||||
return this with { FetchCache = new Dictionary<string, UbuntuFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) };
|
||||
}
|
||||
|
||||
public UbuntuCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids)
|
||||
=> this with
|
||||
{
|
||||
LastPublished = published.ToUniversalTime(),
|
||||
ProcessedNoticeIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? EmptyIds
|
||||
};
|
||||
|
||||
public bool TryGetCache(string key, out UbuntuFetchCacheEntry entry)
|
||||
{
|
||||
if (FetchCache.Count == 0)
|
||||
{
|
||||
entry = UbuntuFetchCacheEntry.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
return FetchCache.TryGetValue(key, out entry!);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyIds;
|
||||
}
|
||||
|
||||
var list = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element.BsonType == BsonType.String)
|
||||
{
|
||||
var str = element.AsString.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(str))
|
||||
{
|
||||
list.Add(str);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var list = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
list.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, UbuntuFetchCacheEntry> ReadCache(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDoc || cacheDoc.ElementCount == 0)
|
||||
{
|
||||
return EmptyCache;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, UbuntuFetchCacheEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var element in cacheDoc.Elements)
|
||||
{
|
||||
if (element.Value is BsonDocument entryDoc)
|
||||
{
|
||||
cache[element.Name] = UbuntuFetchCacheEntry.FromBson(entryDoc);
|
||||
}
|
||||
}
|
||||
|
||||
return cache;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
|
||||
|
||||
internal sealed record UbuntuFetchCacheEntry(string? ETag, DateTimeOffset? LastModified)
|
||||
{
|
||||
public static UbuntuFetchCacheEntry Empty { get; } = new(null, null);
|
||||
|
||||
public static UbuntuFetchCacheEntry FromDocument(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document)
|
||||
=> new(document.Etag, document.LastModified);
|
||||
|
||||
public static UbuntuFetchCacheEntry FromBson(BsonDocument document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
string? etag = null;
|
||||
DateTimeOffset? lastModified = null;
|
||||
|
||||
if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String)
|
||||
{
|
||||
etag = etagValue.AsString;
|
||||
}
|
||||
|
||||
if (document.TryGetValue("lastModified", out var modifiedValue))
|
||||
{
|
||||
lastModified = modifiedValue.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
return new UbuntuFetchCacheEntry(etag, lastModified);
|
||||
}
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var doc = new BsonDocument();
|
||||
if (!string.IsNullOrWhiteSpace(ETag))
|
||||
{
|
||||
doc["etag"] = ETag;
|
||||
}
|
||||
|
||||
if (LastModified.HasValue)
|
||||
{
|
||||
doc["lastModified"] = LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
public bool Matches(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document)
|
||||
{
|
||||
if (document is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!string.Equals(ETag, document.Etag, StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (LastModified.HasValue && document.LastModified.HasValue)
|
||||
{
|
||||
return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return !LastModified.HasValue && !document.LastModified.HasValue;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Distro;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
|
||||
|
||||
internal static class UbuntuMapper
|
||||
{
|
||||
public static Advisory Map(UbuntuNoticeDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(
|
||||
UbuntuConnectorPlugin.SourceName,
|
||||
"document",
|
||||
document.Uri,
|
||||
document.FetchedAt.ToUniversalTime());
|
||||
|
||||
var mapProvenance = new AdvisoryProvenance(
|
||||
UbuntuConnectorPlugin.SourceName,
|
||||
"mapping",
|
||||
dto.NoticeId,
|
||||
recordedAt);
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.NoticeId,
|
||||
title: dto.Title ?? dto.NoticeId,
|
||||
summary: dto.Summary,
|
||||
language: "en",
|
||||
published: dto.Published,
|
||||
modified: recordedAt > dto.Published ? recordedAt : dto.Published,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mapProvenance });
|
||||
}
|
||||
|
||||
private static string[] BuildAliases(UbuntuNoticeDto dto)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
dto.NoticeId
|
||||
};
|
||||
|
||||
foreach (var cve in dto.CveIds ?? Array.Empty<string>())
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
private static AdvisoryReference[] BuildReferences(UbuntuNoticeDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.References is null || dto.References.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>(dto.References.Count);
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
UbuntuConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt);
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url.Trim(),
|
||||
NormalizeReferenceKind(reference.Kind),
|
||||
reference.Kind,
|
||||
reference.Title,
|
||||
provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore poorly formed URIs
|
||||
}
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<AdvisoryReference>()
|
||||
: references
|
||||
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceKind(string? kind)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kind))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return kind.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"external" => "external",
|
||||
"self" => "advisory",
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(UbuntuNoticeDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Packages is null || dto.Packages.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var list = new List<AffectedPackage>();
|
||||
foreach (var package in dto.Packages)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(package.Package) || string.IsNullOrWhiteSpace(package.Version))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DebianEvr.TryParse(package.Version, out var evr) || evr is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
UbuntuConnectorPlugin.SourceName,
|
||||
"affected",
|
||||
$"{dto.NoticeId}:{package.Release}:{package.Package}",
|
||||
recordedAt);
|
||||
|
||||
var rangeProvenance = new AdvisoryProvenance(
|
||||
UbuntuConnectorPlugin.SourceName,
|
||||
"range",
|
||||
$"{dto.NoticeId}:{package.Release}:{package.Package}",
|
||||
recordedAt);
|
||||
|
||||
var rangeExpression = $"fixed:{package.Version}";
|
||||
|
||||
var extensions = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["ubuntu.release"] = package.Release,
|
||||
["ubuntu.pocket"] = package.Pocket ?? string.Empty
|
||||
};
|
||||
|
||||
var range = new AffectedVersionRange(
|
||||
rangeKind: "evr",
|
||||
introducedVersion: null,
|
||||
fixedVersion: package.Version,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: rangeExpression,
|
||||
provenance: rangeProvenance,
|
||||
primitives: new RangePrimitives(
|
||||
SemVer: null,
|
||||
Nevra: null,
|
||||
Evr: new EvrPrimitive(
|
||||
Introduced: null,
|
||||
Fixed: new EvrComponent(evr.Epoch, evr.Version, evr.Revision.Length == 0 ? null : evr.Revision),
|
||||
LastAffected: null),
|
||||
VendorExtensions: extensions));
|
||||
|
||||
var statuses = new[]
|
||||
{
|
||||
new AffectedPackageStatus(DetermineStatus(package), provenance)
|
||||
};
|
||||
|
||||
var normalizedNote = string.IsNullOrWhiteSpace(package.Release)
|
||||
? null
|
||||
: $"ubuntu:{package.Release.Trim()}";
|
||||
var normalizedRule = range.ToNormalizedVersionRule(normalizedNote);
|
||||
var normalizedVersions = normalizedRule is null
|
||||
? Array.Empty<NormalizedVersionRule>()
|
||||
: new[] { normalizedRule };
|
||||
|
||||
list.Add(new AffectedPackage(
|
||||
type: AffectedPackageTypes.Deb,
|
||||
identifier: package.Package,
|
||||
platform: package.Release,
|
||||
versionRanges: new[] { range },
|
||||
statuses: statuses,
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: normalizedVersions));
|
||||
}
|
||||
|
||||
return list.Count == 0
|
||||
? Array.Empty<AffectedPackage>()
|
||||
: list
|
||||
.OrderBy(static pkg => pkg.Platform, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static pkg => pkg.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string DetermineStatus(UbuntuReleasePackageDto package)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(package.Pocket) && package.Pocket.Contains("security", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "resolved";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(package.Pocket) && package.Pocket.Contains("esm", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "resolved";
|
||||
}
|
||||
|
||||
return "resolved";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
|
||||
|
||||
internal sealed record UbuntuNoticeDto(
|
||||
string NoticeId,
|
||||
DateTimeOffset Published,
|
||||
string Title,
|
||||
string Summary,
|
||||
IReadOnlyList<string> CveIds,
|
||||
IReadOnlyList<UbuntuReleasePackageDto> Packages,
|
||||
IReadOnlyList<UbuntuReferenceDto> References);
|
||||
|
||||
internal sealed record UbuntuReleasePackageDto(
|
||||
string Release,
|
||||
string Package,
|
||||
string Version,
|
||||
string Pocket,
|
||||
bool IsSource);
|
||||
|
||||
internal sealed record UbuntuReferenceDto(
|
||||
string Url,
|
||||
string? Kind,
|
||||
string? Title);
|
||||
@@ -0,0 +1,215 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
|
||||
|
||||
internal static class UbuntuNoticeParser
|
||||
{
|
||||
public static UbuntuIndexResponse ParseIndex(string json)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(json);
|
||||
|
||||
using var document = JsonDocument.Parse(json);
|
||||
var root = document.RootElement;
|
||||
if (!root.TryGetProperty("notices", out var noticesElement) || noticesElement.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return UbuntuIndexResponse.Empty;
|
||||
}
|
||||
|
||||
var notices = new List<UbuntuNoticeDto>(noticesElement.GetArrayLength());
|
||||
foreach (var noticeElement in noticesElement.EnumerateArray())
|
||||
{
|
||||
if (!noticeElement.TryGetProperty("id", out var idElement))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var noticeId = idElement.GetString();
|
||||
if (string.IsNullOrWhiteSpace(noticeId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var published = ParseDate(noticeElement, "published") ?? DateTimeOffset.UtcNow;
|
||||
var title = noticeElement.TryGetProperty("title", out var titleElement)
|
||||
? titleElement.GetString() ?? noticeId
|
||||
: noticeId;
|
||||
|
||||
var summary = noticeElement.TryGetProperty("summary", out var summaryElement)
|
||||
? summaryElement.GetString() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
var cves = ExtractCves(noticeElement);
|
||||
var references = ExtractReferences(noticeElement);
|
||||
var packages = ExtractPackages(noticeElement);
|
||||
|
||||
if (packages.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
notices.Add(new UbuntuNoticeDto(
|
||||
noticeId,
|
||||
published,
|
||||
title,
|
||||
summary,
|
||||
cves,
|
||||
packages,
|
||||
references));
|
||||
}
|
||||
|
||||
var offset = root.TryGetProperty("offset", out var offsetElement) && offsetElement.ValueKind == JsonValueKind.Number
|
||||
? offsetElement.GetInt32()
|
||||
: 0;
|
||||
|
||||
var limit = root.TryGetProperty("limit", out var limitElement) && limitElement.ValueKind == JsonValueKind.Number
|
||||
? limitElement.GetInt32()
|
||||
: noticesElement.GetArrayLength();
|
||||
|
||||
var totalResults = root.TryGetProperty("total_results", out var totalElement) && totalElement.ValueKind == JsonValueKind.Number
|
||||
? totalElement.GetInt32()
|
||||
: notices.Count;
|
||||
|
||||
return new UbuntuIndexResponse(offset, limit, totalResults, notices);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCves(JsonElement noticeElement)
|
||||
{
|
||||
if (!noticeElement.TryGetProperty("cves", out var cveArray) || cveArray.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var cveElement in cveArray.EnumerateArray())
|
||||
{
|
||||
var cve = cveElement.TryGetProperty("id", out var idElement)
|
||||
? idElement.GetString()
|
||||
: cveElement.GetString();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
set.Add(cve.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (set.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var list = new List<string>(set);
|
||||
list.Sort(StringComparer.OrdinalIgnoreCase);
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<UbuntuReferenceDto> ExtractReferences(JsonElement noticeElement)
|
||||
{
|
||||
if (!noticeElement.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return Array.Empty<UbuntuReferenceDto>();
|
||||
}
|
||||
|
||||
var list = new List<UbuntuReferenceDto>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var referenceElement in referencesElement.EnumerateArray())
|
||||
{
|
||||
var url = referenceElement.TryGetProperty("url", out var urlElement)
|
||||
? urlElement.GetString()
|
||||
: null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(url) || !seen.Add(url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var kind = referenceElement.TryGetProperty("category", out var categoryElement)
|
||||
? categoryElement.GetString()
|
||||
: null;
|
||||
|
||||
var title = referenceElement.TryGetProperty("summary", out var summaryElement)
|
||||
? summaryElement.GetString()
|
||||
: null;
|
||||
|
||||
list.Add(new UbuntuReferenceDto(url.Trim(), kind, title));
|
||||
}
|
||||
|
||||
return list.Count == 0 ? Array.Empty<UbuntuReferenceDto>() : list;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<UbuntuReleasePackageDto> ExtractPackages(JsonElement noticeElement)
|
||||
{
|
||||
if (!noticeElement.TryGetProperty("release_packages", out var releasesElement) || releasesElement.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return Array.Empty<UbuntuReleasePackageDto>();
|
||||
}
|
||||
|
||||
var packages = new List<UbuntuReleasePackageDto>();
|
||||
foreach (var releaseProperty in releasesElement.EnumerateObject())
|
||||
{
|
||||
var release = releaseProperty.Name;
|
||||
var packageArray = releaseProperty.Value;
|
||||
if (packageArray.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var packageElement in packageArray.EnumerateArray())
|
||||
{
|
||||
var name = packageElement.TryGetProperty("name", out var nameElement)
|
||||
? nameElement.GetString()
|
||||
: null;
|
||||
|
||||
var version = packageElement.TryGetProperty("version", out var versionElement)
|
||||
? versionElement.GetString()
|
||||
: null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(version))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var pocket = packageElement.TryGetProperty("pocket", out var pocketElement)
|
||||
? pocketElement.GetString() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
var isSource = packageElement.TryGetProperty("is_source", out var sourceElement)
|
||||
&& sourceElement.ValueKind == JsonValueKind.True;
|
||||
|
||||
packages.Add(new UbuntuReleasePackageDto(
|
||||
release,
|
||||
name.Trim(),
|
||||
version.Trim(),
|
||||
pocket.Trim(),
|
||||
isSource));
|
||||
}
|
||||
}
|
||||
|
||||
return packages.Count == 0 ? Array.Empty<UbuntuReleasePackageDto>() : packages;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(JsonElement element, string propertyName)
|
||||
{
|
||||
if (!element.TryGetProperty(propertyName, out var dateElement) || dateElement.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var value = dateElement.GetString();
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)
|
||||
? parsed.ToUniversalTime()
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record UbuntuIndexResponse(int Offset, int Limit, int TotalResults, IReadOnlyList<UbuntuNoticeDto> Notices)
|
||||
{
|
||||
public static UbuntuIndexResponse Empty { get; } = new(0, 0, 0, Array.Empty<UbuntuNoticeDto>());
|
||||
}
|
||||
46
src/StellaOps.Concelier.Connector.Distro.Ubuntu/Jobs.cs
Normal file
46
src/StellaOps.Concelier.Connector.Distro.Ubuntu/Jobs.cs
Normal file
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
|
||||
|
||||
internal static class UbuntuJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ubuntu:fetch";
|
||||
public const string Parse = "source:ubuntu:parse";
|
||||
public const string Map = "source:ubuntu:map";
|
||||
}
|
||||
|
||||
internal sealed class UbuntuFetchJob : IJob
|
||||
{
|
||||
private readonly UbuntuConnector _connector;
|
||||
|
||||
public UbuntuFetchJob(UbuntuConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class UbuntuParseJob : IJob
|
||||
{
|
||||
private readonly UbuntuConnector _connector;
|
||||
|
||||
public UbuntuParseJob(UbuntuConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class UbuntuMapJob : IJob
|
||||
{
|
||||
private readonly UbuntuConnector _connector;
|
||||
|
||||
public UbuntuMapJob(UbuntuConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
10
src/StellaOps.Concelier.Connector.Distro.Ubuntu/TASKS.md
Normal file
10
src/StellaOps.Concelier.Connector.Distro.Ubuntu/TASKS.md
Normal file
@@ -0,0 +1,10 @@
|
||||
# Ubuntu Connector TODOs
|
||||
|
||||
| Task | Status | Notes |
|
||||
|---|---|---|
|
||||
|Discover data model & pagination for `notices.json`|DONE|Connector now walks `offset`/`limit` pages (configurable page size) until MaxNoticesPerFetch satisfied, reusing cached pages when unchanged.|
|
||||
|Design cursor & state model|DONE|Cursor tracks last published timestamp plus processed USN identifiers with overlap logic.|
|
||||
|Implement fetch/parse pipeline|DONE|Index fetch hydrates per-notice DTOs, stores metadata, and maps without dedicated detail fetches.|
|
||||
|Emit RangePrimitives + telemetry|DONE|Each package emits EVR primitives with `ubuntu.release` and `ubuntu.pocket` extensions for dashboards.|
|
||||
|Add integration tests|DONE|Fixture-driven fetch→map suite covers resolved and ESM pockets, including conditional GET behaviour.|
|
||||
|NormalizedVersions rollout|DONE (2025-10-11)|EVR ranges now project `normalizedVersions` with `ubuntu:<release>` notes; tests assert canonical rule emission.|
|
||||
@@ -0,0 +1,537 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
|
||||
using StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
|
||||
|
||||
public sealed class UbuntuConnector : IFeedConnector
|
||||
{
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly UbuntuOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<UbuntuConnector> _logger;
|
||||
|
||||
private static readonly Action<ILogger, string, int, Exception?> LogMapped =
|
||||
LoggerMessage.Define<string, int>(
|
||||
LogLevel.Information,
|
||||
new EventId(1, "UbuntuMapped"),
|
||||
"Ubuntu notice {NoticeId} mapped with {PackageCount} packages");
|
||||
|
||||
public UbuntuConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<UbuntuOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<UbuntuConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => UbuntuConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var fetchCache = new Dictionary<string, UbuntuFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase);
|
||||
var pendingMappings = new HashSet<Guid>(cursor.PendingMappings);
|
||||
var processedIds = new HashSet<string>(cursor.ProcessedNoticeIds, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var indexResult = await FetchIndexAsync(cursor, fetchCache, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (indexResult.IsUnchanged)
|
||||
{
|
||||
await UpdateCursorAsync(cursor.WithFetchCache(fetchCache), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (indexResult.Notices.Count == 0)
|
||||
{
|
||||
await UpdateCursorAsync(cursor.WithFetchCache(fetchCache), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var notices = indexResult.Notices;
|
||||
|
||||
var baseline = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap;
|
||||
if (baseline < DateTimeOffset.UnixEpoch)
|
||||
{
|
||||
baseline = DateTimeOffset.UnixEpoch;
|
||||
}
|
||||
|
||||
ProvenanceDiagnostics.ReportResumeWindow(SourceName, baseline, _logger);
|
||||
|
||||
var candidates = notices
|
||||
.Where(notice => notice.Published >= baseline)
|
||||
.OrderBy(notice => notice.Published)
|
||||
.ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
candidates = notices
|
||||
.OrderByDescending(notice => notice.Published)
|
||||
.ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxNoticesPerFetch)
|
||||
.OrderBy(notice => notice.Published)
|
||||
.ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
else if (candidates.Count > _options.MaxNoticesPerFetch)
|
||||
{
|
||||
candidates = candidates
|
||||
.OrderByDescending(notice => notice.Published)
|
||||
.ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxNoticesPerFetch)
|
||||
.OrderBy(notice => notice.Published)
|
||||
.ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
|
||||
var processedWindow = new List<string>(candidates.Count);
|
||||
|
||||
foreach (var notice in candidates)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var detailUri = new Uri(_options.NoticeDetailBaseUri, notice.NoticeId);
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri.AbsoluteUri, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["ubuntu.id"] = notice.NoticeId,
|
||||
["ubuntu.published"] = notice.Published.ToString("O")
|
||||
};
|
||||
|
||||
var dtoDocument = ToBson(notice);
|
||||
var sha256 = ComputeNoticeHash(dtoDocument);
|
||||
|
||||
var documentId = existing?.Id ?? Guid.NewGuid();
|
||||
var record = new DocumentRecord(
|
||||
documentId,
|
||||
SourceName,
|
||||
detailUri.AbsoluteUri,
|
||||
now,
|
||||
sha256,
|
||||
DocumentStatuses.PendingMap,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: existing?.Etag,
|
||||
LastModified: existing?.LastModified ?? notice.Published,
|
||||
GridFsId: null);
|
||||
|
||||
await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), record.Id, SourceName, "ubuntu.notice.v1", dtoDocument, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Add(record.Id);
|
||||
processedIds.Add(notice.NoticeId);
|
||||
processedWindow.Add(notice.NoticeId);
|
||||
|
||||
if (notice.Published > maxPublished)
|
||||
{
|
||||
maxPublished = notice.Published;
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithFetchCache(fetchCache)
|
||||
.WithPendingDocuments(Array.Empty<Guid>())
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithProcessed(maxPublished, processedWindow);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pending = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
pending.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
UbuntuNoticeDto notice;
|
||||
try
|
||||
{
|
||||
notice = FromBson(dto.Payload);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize Ubuntu notice DTO for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pending.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisory = UbuntuMapper.Map(notice, document, _timeProvider.GetUtcNow());
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pending.Remove(documentId);
|
||||
|
||||
LogMapped(_logger, notice.NoticeId, advisory.AffectedPackages.Length, null);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pending);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<UbuntuIndexFetchResult> FetchIndexAsync(
|
||||
UbuntuCursor cursor,
|
||||
IDictionary<string, UbuntuFetchCacheEntry> fetchCache,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var pageSize = Math.Clamp(_options.IndexPageSize, 1, UbuntuOptions.MaxPageSize);
|
||||
var maxNotices = Math.Clamp(_options.MaxNoticesPerFetch, 1, 200);
|
||||
var maxPages = Math.Max(1, (int)Math.Ceiling(maxNotices / (double)pageSize));
|
||||
var aggregated = new List<UbuntuNoticeDto>(Math.Min(maxNotices, pageSize * maxPages));
|
||||
var seenNoticeIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var offset = 0;
|
||||
var totalResults = int.MaxValue;
|
||||
|
||||
for (var pageIndex = 0; pageIndex < maxPages && offset < totalResults; pageIndex++)
|
||||
{
|
||||
var pageUri = BuildIndexUri(_options.NoticesEndpoint, offset, pageSize);
|
||||
var cacheKey = pageUri.ToString();
|
||||
|
||||
cursor.TryGetCache(cacheKey, out var cachedEntry);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["ubuntu.type"] = "index",
|
||||
["ubuntu.offset"] = offset.ToString(CultureInfo.InvariantCulture),
|
||||
["ubuntu.limit"] = pageSize.ToString(CultureInfo.InvariantCulture)
|
||||
};
|
||||
|
||||
var indexRequest = new SourceFetchRequest(UbuntuOptions.HttpClientName, SourceName, pageUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
ETag = cachedEntry?.ETag,
|
||||
LastModified = cachedEntry?.LastModified,
|
||||
TimeoutOverride = _options.FetchTimeout,
|
||||
AcceptHeaders = new[] { "application/json" }
|
||||
};
|
||||
|
||||
SourceFetchResult fetchResult;
|
||||
try
|
||||
{
|
||||
fetchResult = await _fetchService.FetchAsync(indexRequest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Ubuntu notices index fetch failed for {Uri}", pageUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
|
||||
if (fetchResult.IsNotModified)
|
||||
{
|
||||
if (pageIndex == 0)
|
||||
{
|
||||
if (cursor.FetchCache.TryGetValue(cacheKey, out var existingCache))
|
||||
{
|
||||
fetchCache[cacheKey] = existingCache;
|
||||
}
|
||||
|
||||
return UbuntuIndexFetchResult.Unchanged();
|
||||
}
|
||||
|
||||
if (!cursor.FetchCache.TryGetValue(cacheKey, out var cachedEntryForPage))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
fetchCache[cacheKey] = cachedEntryForPage;
|
||||
|
||||
var existingDocument = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false);
|
||||
if (existingDocument is null || !existingDocument.GridFsId.HasValue)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
payload = await _rawDocumentStorage.DownloadAsync(existingDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!fetchResult.IsSuccess || fetchResult.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
fetchCache[cacheKey] = UbuntuFetchCacheEntry.FromDocument(fetchResult.Document);
|
||||
|
||||
if (!fetchResult.Document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Ubuntu index document {DocumentId} missing GridFS payload", fetchResult.Document.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
payload = await _rawDocumentStorage.DownloadAsync(fetchResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var page = UbuntuNoticeParser.ParseIndex(Encoding.UTF8.GetString(payload));
|
||||
|
||||
if (page.TotalResults > 0)
|
||||
{
|
||||
totalResults = page.TotalResults;
|
||||
}
|
||||
|
||||
foreach (var notice in page.Notices)
|
||||
{
|
||||
if (!seenNoticeIds.Add(notice.NoticeId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
aggregated.Add(notice);
|
||||
if (aggregated.Count >= maxNotices)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (aggregated.Count >= maxNotices)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (page.Notices.Count < pageSize)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
offset += pageSize;
|
||||
}
|
||||
|
||||
return new UbuntuIndexFetchResult(false, aggregated);
|
||||
}
|
||||
|
||||
private static Uri BuildIndexUri(Uri endpoint, int offset, int limit)
|
||||
{
|
||||
var builder = new UriBuilder(endpoint);
|
||||
var queryBuilder = new StringBuilder();
|
||||
|
||||
if (!string.IsNullOrEmpty(builder.Query))
|
||||
{
|
||||
var existing = builder.Query.TrimStart('?');
|
||||
if (!string.IsNullOrEmpty(existing))
|
||||
{
|
||||
queryBuilder.Append(existing);
|
||||
if (existing[^1] != '&')
|
||||
{
|
||||
queryBuilder.Append('&');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
queryBuilder.Append("offset=");
|
||||
queryBuilder.Append(offset.ToString(CultureInfo.InvariantCulture));
|
||||
queryBuilder.Append("&limit=");
|
||||
queryBuilder.Append(limit.ToString(CultureInfo.InvariantCulture));
|
||||
|
||||
builder.Query = queryBuilder.ToString();
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private sealed record UbuntuIndexFetchResult(bool IsUnchanged, IReadOnlyList<UbuntuNoticeDto> Notices)
|
||||
{
|
||||
public static UbuntuIndexFetchResult Unchanged()
|
||||
=> new(true, Array.Empty<UbuntuNoticeDto>());
|
||||
}
|
||||
|
||||
private async Task<UbuntuCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? UbuntuCursor.Empty : UbuntuCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(UbuntuCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var doc = cursor.ToBsonDocument();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, doc, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string ComputeNoticeHash(BsonDocument document)
|
||||
{
|
||||
var bytes = document.ToBson();
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static BsonDocument ToBson(UbuntuNoticeDto notice)
|
||||
{
|
||||
var packages = new BsonArray();
|
||||
foreach (var package in notice.Packages)
|
||||
{
|
||||
packages.Add(new BsonDocument
|
||||
{
|
||||
["release"] = package.Release,
|
||||
["package"] = package.Package,
|
||||
["version"] = package.Version,
|
||||
["pocket"] = package.Pocket,
|
||||
["isSource"] = package.IsSource
|
||||
});
|
||||
}
|
||||
|
||||
var references = new BsonArray();
|
||||
foreach (var reference in notice.References)
|
||||
{
|
||||
var doc = new BsonDocument
|
||||
{
|
||||
["url"] = reference.Url
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(reference.Kind))
|
||||
{
|
||||
doc["kind"] = reference.Kind;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(reference.Title))
|
||||
{
|
||||
doc["title"] = reference.Title;
|
||||
}
|
||||
|
||||
references.Add(doc);
|
||||
}
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
["noticeId"] = notice.NoticeId,
|
||||
["published"] = notice.Published.UtcDateTime,
|
||||
["title"] = notice.Title,
|
||||
["summary"] = notice.Summary,
|
||||
["cves"] = new BsonArray(notice.CveIds ?? Array.Empty<string>()),
|
||||
["packages"] = packages,
|
||||
["references"] = references
|
||||
};
|
||||
}
|
||||
|
||||
private static UbuntuNoticeDto FromBson(BsonDocument document)
|
||||
{
|
||||
var noticeId = document.GetValue("noticeId", string.Empty).AsString;
|
||||
var published = document.TryGetValue("published", out var publishedValue)
|
||||
? publishedValue.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => DateTimeOffset.UtcNow
|
||||
}
|
||||
: DateTimeOffset.UtcNow;
|
||||
|
||||
var title = document.GetValue("title", noticeId).AsString;
|
||||
var summary = document.GetValue("summary", string.Empty).AsString;
|
||||
|
||||
var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cveBson
|
||||
? cveBson.OfType<BsonValue>()
|
||||
.Select(static value => value?.ToString())
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value!)
|
||||
.ToArray()
|
||||
: Array.Empty<string>();
|
||||
|
||||
var packages = new List<UbuntuReleasePackageDto>();
|
||||
if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packageBson)
|
||||
{
|
||||
foreach (var element in packageBson.OfType<BsonDocument>())
|
||||
{
|
||||
packages.Add(new UbuntuReleasePackageDto(
|
||||
Release: element.GetValue("release", string.Empty).AsString,
|
||||
Package: element.GetValue("package", string.Empty).AsString,
|
||||
Version: element.GetValue("version", string.Empty).AsString,
|
||||
Pocket: element.GetValue("pocket", string.Empty).AsString,
|
||||
IsSource: element.TryGetValue("isSource", out var sourceValue) && sourceValue.AsBoolean));
|
||||
}
|
||||
}
|
||||
|
||||
var references = new List<UbuntuReferenceDto>();
|
||||
if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray referenceBson)
|
||||
{
|
||||
foreach (var element in referenceBson.OfType<BsonDocument>())
|
||||
{
|
||||
var url = element.GetValue("url", string.Empty).AsString;
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
references.Add(new UbuntuReferenceDto(
|
||||
url,
|
||||
element.TryGetValue("kind", out var kindValue) ? kindValue.AsString : null,
|
||||
element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null));
|
||||
}
|
||||
}
|
||||
|
||||
return new UbuntuNoticeDto(
|
||||
noticeId,
|
||||
published,
|
||||
title,
|
||||
summary,
|
||||
cves,
|
||||
packages,
|
||||
references);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
|
||||
|
||||
public sealed class UbuntuConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "distro-ubuntu";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<UbuntuConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
|
||||
|
||||
public sealed class UbuntuDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ubuntu";
|
||||
private const string FetchCron = "*/20 * * * *";
|
||||
private const string ParseCron = "7,27,47 * * * *";
|
||||
private const string MapCron = "10,30,50 * * * *";
|
||||
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4);
|
||||
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(8);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddUbuntuConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler
|
||||
.AddJob<UbuntuFetchJob>(
|
||||
UbuntuJobKinds.Fetch,
|
||||
cronExpression: FetchCron,
|
||||
timeout: FetchTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<UbuntuParseJob>(
|
||||
UbuntuJobKinds.Parse,
|
||||
cronExpression: ParseCron,
|
||||
timeout: ParseTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<UbuntuMapJob>(
|
||||
UbuntuJobKinds.Map,
|
||||
cronExpression: MapCron,
|
||||
timeout: MapTimeout,
|
||||
leaseDuration: LeaseDuration);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
|
||||
|
||||
public static class UbuntuServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddUbuntuConnector(this IServiceCollection services, Action<UbuntuOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<UbuntuOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(UbuntuOptions.HttpClientName, (sp, httpOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<UbuntuOptions>>().Value;
|
||||
httpOptions.BaseAddress = options.NoticesEndpoint.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority
|
||||
? new Uri(authority)
|
||||
: new Uri("https://ubuntu.com/");
|
||||
httpOptions.Timeout = options.FetchTimeout;
|
||||
httpOptions.UserAgent = options.UserAgent;
|
||||
httpOptions.AllowedHosts.Clear();
|
||||
httpOptions.AllowedHosts.Add(options.NoticesEndpoint.Host);
|
||||
httpOptions.AllowedHosts.Add(options.NoticeDetailBaseUri.Host);
|
||||
httpOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
});
|
||||
|
||||
services.AddTransient<UbuntuConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user