Initial commit (history squashed)
Some checks failed
Build Test Deploy / authority-container (push) Has been cancelled
Build Test Deploy / docs (push) Has been cancelled
Build Test Deploy / deploy (push) Has been cancelled
Build Test Deploy / build-test (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Build Test Deploy / authority-container (push) Has been cancelled
Build Test Deploy / docs (push) Has been cancelled
Build Test Deploy / deploy (push) Has been cancelled
Build Test Deploy / build-test (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Debian.Tests")]
|
||||
@@ -0,0 +1,87 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Configuration;
|
||||
|
||||
public sealed class DebianOptions
|
||||
{
|
||||
public const string HttpClientName = "feedser.debian";
|
||||
|
||||
/// <summary>
|
||||
/// Raw advisory list published by the Debian security tracker team.
|
||||
/// Defaults to the Salsa Git raw endpoint to avoid HTML scraping.
|
||||
/// </summary>
|
||||
public Uri ListEndpoint { get; set; } = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list");
|
||||
|
||||
/// <summary>
|
||||
/// Base URI for advisory detail pages. Connector appends {AdvisoryId}.
|
||||
/// </summary>
|
||||
public Uri DetailBaseUri { get; set; } = new("https://security-tracker.debian.org/tracker/");
|
||||
|
||||
/// <summary>
|
||||
/// Maximum advisories fetched per run to cap backfill effort.
|
||||
/// </summary>
|
||||
public int MaxAdvisoriesPerFetch { get; set; } = 40;
|
||||
|
||||
/// <summary>
|
||||
/// Initial history window pulled on first run.
|
||||
/// </summary>
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
/// <summary>
|
||||
/// Resume overlap to accommodate late edits of existing advisories.
|
||||
/// </summary>
|
||||
public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
/// <summary>
|
||||
/// Request timeout used for list/detail fetches unless overridden via HTTP client.
|
||||
/// </summary>
|
||||
public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45);
|
||||
|
||||
/// <summary>
|
||||
/// Optional pacing delay between detail fetches.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero;
|
||||
|
||||
/// <summary>
|
||||
/// Custom user-agent for Debian tracker courtesy.
|
||||
/// </summary>
|
||||
public string UserAgent { get; set; } = "StellaOps.Feedser.Debian/0.1 (+https://stella-ops.org)";
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (ListEndpoint is null || !ListEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Debian list endpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (DetailBaseUri is null || !DetailBaseUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Debian detail base URI must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (MaxAdvisoriesPerFetch <= 0 || MaxAdvisoriesPerFetch > 200)
|
||||
{
|
||||
throw new InvalidOperationException("MaxAdvisoriesPerFetch must be between 1 and 200.");
|
||||
}
|
||||
|
||||
if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365))
|
||||
{
|
||||
throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days.");
|
||||
}
|
||||
|
||||
if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14))
|
||||
{
|
||||
throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days.");
|
||||
}
|
||||
|
||||
if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5))
|
||||
{
|
||||
throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10))
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay must be between 0 and 10 seconds.");
|
||||
}
|
||||
}
|
||||
}
|
||||
637
src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs
Normal file
637
src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs
Normal file
@@ -0,0 +1,637 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
using StellaOps.Feedser.Source.Distro.Debian.Configuration;
|
||||
using StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian;
|
||||
|
||||
public sealed class DebianConnector : IFeedConnector
|
||||
{
|
||||
private const string SchemaVersion = "debian.v1";
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly DebianOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<DebianConnector> _logger;
|
||||
|
||||
private static readonly Action<ILogger, string, int, Exception?> LogMapped =
|
||||
LoggerMessage.Define<string, int>(
|
||||
LogLevel.Information,
|
||||
new EventId(1, "DebianMapped"),
|
||||
"Debian advisory {AdvisoryId} mapped with {AffectedCount} packages");
|
||||
|
||||
public DebianConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<DebianOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<DebianConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => DebianConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments);
|
||||
var pendingMappings = new HashSet<Guid>(cursor.PendingMappings);
|
||||
var fetchCache = new Dictionary<string, DebianFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase);
|
||||
var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var listUri = _options.ListEndpoint;
|
||||
var listKey = listUri.ToString();
|
||||
touchedResources.Add(listKey);
|
||||
|
||||
var existingList = await _documentStore.FindBySourceAndUriAsync(SourceName, listKey, cancellationToken).ConfigureAwait(false);
|
||||
cursor.TryGetCache(listKey, out var cachedListEntry);
|
||||
|
||||
var listRequest = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, listUri)
|
||||
{
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["type"] = "index"
|
||||
},
|
||||
AcceptHeaders = new[] { "text/plain", "text/plain; charset=utf-8" },
|
||||
TimeoutOverride = _options.FetchTimeout,
|
||||
ETag = existingList?.Etag ?? cachedListEntry?.ETag,
|
||||
LastModified = existingList?.LastModified ?? cachedListEntry?.LastModified,
|
||||
};
|
||||
|
||||
SourceFetchResult listResult;
|
||||
try
|
||||
{
|
||||
listResult = await _fetchService.FetchAsync(listRequest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Debian list fetch failed");
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
var lastPublished = cursor.LastPublished ?? (now - _options.InitialBackfill);
|
||||
var processedIds = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase);
|
||||
var newProcessedIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
|
||||
var processedUpdated = false;
|
||||
|
||||
if (listResult.IsNotModified)
|
||||
{
|
||||
if (existingList is not null)
|
||||
{
|
||||
fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(existingList);
|
||||
}
|
||||
}
|
||||
else if (listResult.IsSuccess && listResult.Document is not null)
|
||||
{
|
||||
fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(listResult.Document);
|
||||
|
||||
if (!listResult.Document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Debian list document {DocumentId} missing GridFS payload", listResult.Document.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
byte[] bytes;
|
||||
try
|
||||
{
|
||||
bytes = await _rawDocumentStorage.DownloadAsync(listResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to download Debian list document {DocumentId}", listResult.Document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
var text = System.Text.Encoding.UTF8.GetString(bytes);
|
||||
var entries = DebianListParser.Parse(text);
|
||||
if (entries.Count > 0)
|
||||
{
|
||||
var windowStart = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap;
|
||||
if (windowStart < DateTimeOffset.UnixEpoch)
|
||||
{
|
||||
windowStart = DateTimeOffset.UnixEpoch;
|
||||
}
|
||||
|
||||
ProvenanceDiagnostics.ReportResumeWindow(SourceName, windowStart, _logger);
|
||||
|
||||
var candidates = entries
|
||||
.Where(entry => entry.Published >= windowStart)
|
||||
.OrderBy(entry => entry.Published)
|
||||
.ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
candidates = entries
|
||||
.OrderByDescending(entry => entry.Published)
|
||||
.ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxAdvisoriesPerFetch)
|
||||
.OrderBy(entry => entry.Published)
|
||||
.ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
else if (candidates.Count > _options.MaxAdvisoriesPerFetch)
|
||||
{
|
||||
candidates = candidates
|
||||
.OrderByDescending(entry => entry.Published)
|
||||
.ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxAdvisoriesPerFetch)
|
||||
.OrderBy(entry => entry.Published)
|
||||
.ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
foreach (var entry in candidates)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var detailUri = new Uri(_options.DetailBaseUri, entry.AdvisoryId);
|
||||
var cacheKey = detailUri.ToString();
|
||||
touchedResources.Add(cacheKey);
|
||||
|
||||
cursor.TryGetCache(cacheKey, out var cachedDetail);
|
||||
if (!fetchCache.TryGetValue(cacheKey, out var cachedInRun))
|
||||
{
|
||||
cachedInRun = cachedDetail;
|
||||
}
|
||||
|
||||
var metadata = BuildDetailMetadata(entry);
|
||||
var existingDetail = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var request = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, detailUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
AcceptHeaders = new[] { "text/html", "application/xhtml+xml" },
|
||||
TimeoutOverride = _options.FetchTimeout,
|
||||
ETag = existingDetail?.Etag ?? cachedInRun?.ETag,
|
||||
LastModified = existingDetail?.LastModified ?? cachedInRun?.LastModified,
|
||||
};
|
||||
|
||||
SourceFetchResult result;
|
||||
try
|
||||
{
|
||||
result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to fetch Debian advisory {AdvisoryId}", entry.AdvisoryId);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
if (existingDetail is not null)
|
||||
{
|
||||
fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(existingDetail);
|
||||
if (string.Equals(existingDetail.Status, DocumentStatuses.Mapped, StringComparison.Ordinal))
|
||||
{
|
||||
pendingDocuments.Remove(existingDetail.Id);
|
||||
pendingMappings.Remove(existingDetail.Id);
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(result.Document);
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
pendingMappings.Remove(result.Document.Id);
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (entry.Published > maxPublished)
|
||||
{
|
||||
maxPublished = entry.Published;
|
||||
newProcessedIds.Clear();
|
||||
processedUpdated = true;
|
||||
}
|
||||
|
||||
if (entry.Published == maxPublished)
|
||||
{
|
||||
newProcessedIds.Add(entry.AdvisoryId);
|
||||
processedUpdated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fetchCache.Count > 0 && touchedResources.Count > 0)
|
||||
{
|
||||
var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray();
|
||||
foreach (var key in stale)
|
||||
{
|
||||
fetchCache.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
if (!processedUpdated && cursor.LastPublished.HasValue)
|
||||
{
|
||||
maxPublished = cursor.LastPublished.Value;
|
||||
newProcessedIds = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithFetchCache(fetchCache);
|
||||
|
||||
if (processedUpdated && maxPublished > DateTimeOffset.MinValue)
|
||||
{
|
||||
updatedCursor = updatedCursor.WithProcessed(maxPublished, newProcessedIds);
|
||||
}
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remaining = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Debian document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = ExtractMetadata(document);
|
||||
if (metadata is null)
|
||||
{
|
||||
_logger.LogWarning("Debian document {DocumentId} missing required metadata", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] bytes;
|
||||
try
|
||||
{
|
||||
bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to download Debian document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
var html = System.Text.Encoding.UTF8.GetString(bytes);
|
||||
DebianAdvisoryDto dto;
|
||||
try
|
||||
{
|
||||
dto = DebianHtmlParser.Parse(html, metadata);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse Debian advisory {AdvisoryId}", metadata.AdvisoryId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(document.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var payload = ToBson(dto);
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow());
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remaining.Remove(document.Id);
|
||||
if (!pendingMappings.Contains(document.Id))
|
||||
{
|
||||
pendingMappings.Add(document.Id);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remaining)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null || document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
DebianAdvisoryDto dto;
|
||||
try
|
||||
{
|
||||
dto = FromBson(dtoRecord.Payload);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize Debian DTO for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow());
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<DebianCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? DebianCursor.Empty : DebianCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(DebianCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildDetailMetadata(DebianListEntry entry)
|
||||
{
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["debian.id"] = entry.AdvisoryId,
|
||||
["debian.published"] = entry.Published.ToString("O", CultureInfo.InvariantCulture),
|
||||
["debian.title"] = entry.Title,
|
||||
["debian.package"] = entry.SourcePackage
|
||||
};
|
||||
|
||||
if (entry.CveIds.Count > 0)
|
||||
{
|
||||
metadata["debian.cves"] = string.Join(' ', entry.CveIds);
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static DebianDetailMetadata? ExtractMetadata(DocumentRecord document)
|
||||
{
|
||||
if (document.Metadata is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!document.Metadata.TryGetValue("debian.id", out var id) || string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!document.Metadata.TryGetValue("debian.published", out var publishedRaw)
|
||||
|| !DateTimeOffset.TryParse(publishedRaw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published))
|
||||
{
|
||||
published = document.FetchedAt;
|
||||
}
|
||||
|
||||
var title = document.Metadata.TryGetValue("debian.title", out var t) ? t : id;
|
||||
var package = document.Metadata.TryGetValue("debian.package", out var pkg) && !string.IsNullOrWhiteSpace(pkg)
|
||||
? pkg
|
||||
: id;
|
||||
|
||||
IReadOnlyList<string> cveList = Array.Empty<string>();
|
||||
if (document.Metadata.TryGetValue("debian.cves", out var cvesRaw) && !string.IsNullOrWhiteSpace(cvesRaw))
|
||||
{
|
||||
cveList = cvesRaw
|
||||
.Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)
|
||||
.Where(static s => !string.IsNullOrWhiteSpace(s))
|
||||
.Select(static s => s!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
return new DebianDetailMetadata(
|
||||
id.Trim(),
|
||||
new Uri(document.Uri, UriKind.Absolute),
|
||||
published.ToUniversalTime(),
|
||||
title,
|
||||
package,
|
||||
cveList);
|
||||
}
|
||||
|
||||
private static BsonDocument ToBson(DebianAdvisoryDto dto)
|
||||
{
|
||||
var packages = new BsonArray();
|
||||
foreach (var package in dto.Packages)
|
||||
{
|
||||
var packageDoc = new BsonDocument
|
||||
{
|
||||
["package"] = package.Package,
|
||||
["release"] = package.Release,
|
||||
["status"] = package.Status,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(package.IntroducedVersion))
|
||||
{
|
||||
packageDoc["introduced"] = package.IntroducedVersion;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(package.FixedVersion))
|
||||
{
|
||||
packageDoc["fixed"] = package.FixedVersion;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(package.LastAffectedVersion))
|
||||
{
|
||||
packageDoc["last"] = package.LastAffectedVersion;
|
||||
}
|
||||
|
||||
if (package.Published.HasValue)
|
||||
{
|
||||
packageDoc["published"] = package.Published.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
packages.Add(packageDoc);
|
||||
}
|
||||
|
||||
var references = new BsonArray(dto.References.Select(reference =>
|
||||
{
|
||||
var doc = new BsonDocument
|
||||
{
|
||||
["url"] = reference.Url
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(reference.Kind))
|
||||
{
|
||||
doc["kind"] = reference.Kind;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(reference.Title))
|
||||
{
|
||||
doc["title"] = reference.Title;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}));
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
["advisoryId"] = dto.AdvisoryId,
|
||||
["sourcePackage"] = dto.SourcePackage,
|
||||
["title"] = dto.Title,
|
||||
["description"] = dto.Description ?? string.Empty,
|
||||
["cves"] = new BsonArray(dto.CveIds),
|
||||
["packages"] = packages,
|
||||
["references"] = references,
|
||||
};
|
||||
}
|
||||
|
||||
private static DebianAdvisoryDto FromBson(BsonDocument document)
|
||||
{
|
||||
var advisoryId = document.GetValue("advisoryId", "").AsString;
|
||||
var sourcePackage = document.GetValue("sourcePackage", advisoryId).AsString;
|
||||
var title = document.GetValue("title", advisoryId).AsString;
|
||||
var description = document.TryGetValue("description", out var desc) ? desc.AsString : null;
|
||||
|
||||
var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cvesBson
|
||||
? cvesBson.OfType<BsonValue>()
|
||||
.Select(static value => value.ToString())
|
||||
.Where(static s => !string.IsNullOrWhiteSpace(s))
|
||||
.Select(static s => s!)
|
||||
.ToArray()
|
||||
: Array.Empty<string>();
|
||||
|
||||
var packages = new List<DebianPackageStateDto>();
|
||||
if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packagesBson)
|
||||
{
|
||||
foreach (var element in packagesBson.OfType<BsonDocument>())
|
||||
{
|
||||
packages.Add(new DebianPackageStateDto(
|
||||
element.GetValue("package", sourcePackage).AsString,
|
||||
element.GetValue("release", string.Empty).AsString,
|
||||
element.GetValue("status", "unknown").AsString,
|
||||
element.TryGetValue("introduced", out var introducedValue) ? introducedValue.AsString : null,
|
||||
element.TryGetValue("fixed", out var fixedValue) ? fixedValue.AsString : null,
|
||||
element.TryGetValue("last", out var lastValue) ? lastValue.AsString : null,
|
||||
element.TryGetValue("published", out var publishedValue)
|
||||
? publishedValue.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => (DateTimeOffset?)null,
|
||||
}
|
||||
: null));
|
||||
}
|
||||
}
|
||||
|
||||
var references = new List<DebianReferenceDto>();
|
||||
if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray refBson)
|
||||
{
|
||||
foreach (var element in refBson.OfType<BsonDocument>())
|
||||
{
|
||||
references.Add(new DebianReferenceDto(
|
||||
element.GetValue("url", "").AsString,
|
||||
element.TryGetValue("kind", out var kind) ? kind.AsString : null,
|
||||
element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null));
|
||||
}
|
||||
}
|
||||
|
||||
return new DebianAdvisoryDto(
|
||||
advisoryId,
|
||||
sourcePackage,
|
||||
title,
|
||||
description,
|
||||
cves,
|
||||
packages,
|
||||
references);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian;
|
||||
|
||||
public sealed class DebianConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "distro-debian";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<DebianConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Source.Distro.Debian.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian;
|
||||
|
||||
public sealed class DebianDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "feedser:sources:debian";
|
||||
private const string FetchSchedule = "*/30 * * * *";
|
||||
private const string ParseSchedule = "7,37 * * * *";
|
||||
private const string MapSchedule = "12,42 * * * *";
|
||||
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6);
|
||||
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddDebianConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler
|
||||
.AddJob<DebianFetchJob>(
|
||||
DebianJobKinds.Fetch,
|
||||
cronExpression: FetchSchedule,
|
||||
timeout: FetchTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<DebianParseJob>(
|
||||
DebianJobKinds.Parse,
|
||||
cronExpression: ParseSchedule,
|
||||
timeout: ParseTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<DebianMapJob>(
|
||||
DebianJobKinds.Map,
|
||||
cronExpression: MapSchedule,
|
||||
timeout: MapTimeout,
|
||||
leaseDuration: LeaseDuration);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
using StellaOps.Feedser.Source.Distro.Debian.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian;
|
||||
|
||||
public static class DebianServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddDebianConnector(this IServiceCollection services, Action<DebianOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<DebianOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(DebianOptions.HttpClientName, (sp, httpOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<DebianOptions>>().Value;
|
||||
httpOptions.BaseAddress = options.DetailBaseUri.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority
|
||||
? new Uri(authority, UriKind.Absolute)
|
||||
: new Uri("https://security-tracker.debian.org/", UriKind.Absolute);
|
||||
httpOptions.Timeout = options.FetchTimeout;
|
||||
httpOptions.UserAgent = options.UserAgent;
|
||||
httpOptions.AllowedHosts.Clear();
|
||||
httpOptions.AllowedHosts.Add(options.DetailBaseUri.Host);
|
||||
httpOptions.AllowedHosts.Add(options.ListEndpoint.Host);
|
||||
httpOptions.DefaultRequestHeaders["Accept"] = "text/html,application/xhtml+xml,text/plain;q=0.9,application/json;q=0.8";
|
||||
});
|
||||
|
||||
services.AddTransient<DebianConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal sealed record DebianAdvisoryDto(
|
||||
string AdvisoryId,
|
||||
string SourcePackage,
|
||||
string? Title,
|
||||
string? Description,
|
||||
IReadOnlyList<string> CveIds,
|
||||
IReadOnlyList<DebianPackageStateDto> Packages,
|
||||
IReadOnlyList<DebianReferenceDto> References);
|
||||
|
||||
internal sealed record DebianPackageStateDto(
|
||||
string Package,
|
||||
string Release,
|
||||
string Status,
|
||||
string? IntroducedVersion,
|
||||
string? FixedVersion,
|
||||
string? LastAffectedVersion,
|
||||
DateTimeOffset? Published);
|
||||
|
||||
internal sealed record DebianReferenceDto(
|
||||
string Url,
|
||||
string? Kind,
|
||||
string? Title);
|
||||
@@ -0,0 +1,177 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal sealed record DebianCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<string> ProcessedAdvisoryIds,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, DebianFetchCacheEntry> FetchCache)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<string> EmptyIds = Array.Empty<string>();
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, DebianFetchCacheEntry> EmptyCache =
|
||||
new Dictionary<string, DebianFetchCacheEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static DebianCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache);
|
||||
|
||||
public static DebianCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
DateTimeOffset? lastPublished = null;
|
||||
if (document.TryGetValue("lastPublished", out var lastValue))
|
||||
{
|
||||
lastPublished = lastValue.BsonType switch
|
||||
{
|
||||
BsonType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
BsonType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
var processed = ReadStringArray(document, "processedIds");
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var cache = ReadCache(document);
|
||||
|
||||
return new DebianCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache);
|
||||
}
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (ProcessedAdvisoryIds.Count > 0)
|
||||
{
|
||||
document["processedIds"] = new BsonArray(ProcessedAdvisoryIds);
|
||||
}
|
||||
|
||||
if (FetchCache.Count > 0)
|
||||
{
|
||||
var cacheDoc = new BsonDocument();
|
||||
foreach (var (key, entry) in FetchCache)
|
||||
{
|
||||
cacheDoc[key] = entry.ToBsonDocument();
|
||||
}
|
||||
|
||||
document["fetchCache"] = cacheDoc;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public DebianCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public DebianCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public DebianCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids)
|
||||
=> this with
|
||||
{
|
||||
LastPublished = published.ToUniversalTime(),
|
||||
ProcessedAdvisoryIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? EmptyIds
|
||||
};
|
||||
|
||||
public DebianCursor WithFetchCache(IDictionary<string, DebianFetchCacheEntry>? cache)
|
||||
{
|
||||
if (cache is null || cache.Count == 0)
|
||||
{
|
||||
return this with { FetchCache = EmptyCache };
|
||||
}
|
||||
|
||||
return this with { FetchCache = new Dictionary<string, DebianFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) };
|
||||
}
|
||||
|
||||
public bool TryGetCache(string key, out DebianFetchCacheEntry entry)
|
||||
{
|
||||
if (FetchCache.Count == 0)
|
||||
{
|
||||
entry = DebianFetchCacheEntry.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
return FetchCache.TryGetValue(key, out entry!);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyIds;
|
||||
}
|
||||
|
||||
var list = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element.BsonType == BsonType.String)
|
||||
{
|
||||
var str = element.AsString.Trim();
|
||||
if (!string.IsNullOrEmpty(str))
|
||||
{
|
||||
list.Add(str);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var list = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
list.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, DebianFetchCacheEntry> ReadCache(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0)
|
||||
{
|
||||
return EmptyCache;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, DebianFetchCacheEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var element in cacheDocument.Elements)
|
||||
{
|
||||
if (element.Value is BsonDocument entry)
|
||||
{
|
||||
cache[element.Name] = DebianFetchCacheEntry.FromBson(entry);
|
||||
}
|
||||
}
|
||||
|
||||
return cache;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal sealed record DebianDetailMetadata(
|
||||
string AdvisoryId,
|
||||
Uri DetailUri,
|
||||
DateTimeOffset Published,
|
||||
string Title,
|
||||
string SourcePackage,
|
||||
IReadOnlyList<string> CveIds);
|
||||
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastModified)
|
||||
{
|
||||
public static DebianFetchCacheEntry Empty { get; } = new(null, null);
|
||||
|
||||
public static DebianFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document)
|
||||
=> new(document.Etag, document.LastModified);
|
||||
|
||||
public static DebianFetchCacheEntry FromBson(BsonDocument document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
string? etag = null;
|
||||
DateTimeOffset? lastModified = null;
|
||||
|
||||
if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String)
|
||||
{
|
||||
etag = etagValue.AsString;
|
||||
}
|
||||
|
||||
if (document.TryGetValue("lastModified", out var modifiedValue))
|
||||
{
|
||||
lastModified = modifiedValue.BsonType switch
|
||||
{
|
||||
BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
return new DebianFetchCacheEntry(etag, lastModified);
|
||||
}
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument();
|
||||
if (!string.IsNullOrWhiteSpace(ETag))
|
||||
{
|
||||
document["etag"] = ETag;
|
||||
}
|
||||
|
||||
if (LastModified.HasValue)
|
||||
{
|
||||
document["lastModified"] = LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document)
|
||||
{
|
||||
if (document is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!string.Equals(document.Etag, ETag, StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (LastModified.HasValue && document.LastModified.HasValue)
|
||||
{
|
||||
return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return !LastModified.HasValue && !document.LastModified.HasValue;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal static class DebianHtmlParser
|
||||
{
|
||||
public static DebianAdvisoryDto Parse(string html, DebianDetailMetadata metadata)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(html);
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
|
||||
var parser = new HtmlParser();
|
||||
var document = parser.ParseDocument(html);
|
||||
|
||||
var description = ExtractDescription(document) ?? metadata.Title;
|
||||
var references = ExtractReferences(document, metadata);
|
||||
var packages = ExtractPackages(document, metadata.SourcePackage, metadata.Published);
|
||||
|
||||
return new DebianAdvisoryDto(
|
||||
metadata.AdvisoryId,
|
||||
metadata.SourcePackage,
|
||||
metadata.Title,
|
||||
description,
|
||||
metadata.CveIds,
|
||||
packages,
|
||||
references);
|
||||
}
|
||||
|
||||
private static string? ExtractDescription(IHtmlDocument document)
|
||||
{
|
||||
foreach (var table in document.QuerySelectorAll("table"))
|
||||
{
|
||||
if (table is not IHtmlTableElement tableElement)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var row in tableElement.Rows)
|
||||
{
|
||||
if (row.Cells.Length < 2)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var header = row.Cells[0].TextContent?.Trim();
|
||||
if (string.Equals(header, "Description", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return NormalizeWhitespace(row.Cells[1].TextContent);
|
||||
}
|
||||
}
|
||||
|
||||
// Only the first table contains the metadata rows we need.
|
||||
break;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<DebianReferenceDto> ExtractReferences(IHtmlDocument document, DebianDetailMetadata metadata)
|
||||
{
|
||||
var references = new List<DebianReferenceDto>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Add canonical Debian advisory page.
|
||||
var canonical = new Uri($"https://www.debian.org/security/{metadata.AdvisoryId.ToLowerInvariant()}");
|
||||
references.Add(new DebianReferenceDto(canonical.ToString(), "advisory", metadata.Title));
|
||||
seen.Add(canonical.ToString());
|
||||
|
||||
foreach (var link in document.QuerySelectorAll("a"))
|
||||
{
|
||||
var href = link.GetAttribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string resolved;
|
||||
if (Uri.TryCreate(href, UriKind.Absolute, out var absolute))
|
||||
{
|
||||
resolved = absolute.ToString();
|
||||
}
|
||||
else if (Uri.TryCreate(metadata.DetailUri, href, out var relative))
|
||||
{
|
||||
resolved = relative.ToString();
|
||||
}
|
||||
else
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seen.Add(resolved))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var text = NormalizeWhitespace(link.TextContent);
|
||||
string? kind = null;
|
||||
if (text.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
kind = "cve";
|
||||
}
|
||||
else if (resolved.Contains("debian.org/security", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
kind = "advisory";
|
||||
}
|
||||
|
||||
references.Add(new DebianReferenceDto(resolved, kind, text));
|
||||
}
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<DebianPackageStateDto> ExtractPackages(IHtmlDocument document, string defaultPackage, DateTimeOffset published)
|
||||
{
|
||||
var table = FindPackagesTable(document);
|
||||
if (table is null)
|
||||
{
|
||||
return Array.Empty<DebianPackageStateDto>();
|
||||
}
|
||||
|
||||
var accumulators = new Dictionary<string, PackageAccumulator>(StringComparer.OrdinalIgnoreCase);
|
||||
string currentPackage = defaultPackage;
|
||||
|
||||
foreach (var body in table.Bodies)
|
||||
{
|
||||
foreach (var row in body.Rows)
|
||||
{
|
||||
if (row.Cells.Length < 4)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var packageCell = NormalizeWhitespace(row.Cells[0].TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(packageCell))
|
||||
{
|
||||
currentPackage = ExtractPackageName(packageCell);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(currentPackage))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var releaseRaw = NormalizeWhitespace(row.Cells[1].TextContent);
|
||||
var versionRaw = NormalizeWhitespace(row.Cells[2].TextContent);
|
||||
var statusRaw = NormalizeWhitespace(row.Cells[3].TextContent);
|
||||
if (string.IsNullOrWhiteSpace(releaseRaw))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var release = NormalizeRelease(releaseRaw);
|
||||
var key = $"{currentPackage}|{release}";
|
||||
if (!accumulators.TryGetValue(key, out var accumulator))
|
||||
{
|
||||
accumulator = new PackageAccumulator(currentPackage, release, published);
|
||||
accumulators[key] = accumulator;
|
||||
}
|
||||
|
||||
accumulator.Apply(statusRaw, versionRaw);
|
||||
}
|
||||
}
|
||||
|
||||
return accumulators.Values
|
||||
.Where(static acc => acc.ShouldEmit)
|
||||
.Select(static acc => acc.ToDto())
|
||||
.OrderBy(static dto => dto.Release, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static dto => dto.Package, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IHtmlTableElement? FindPackagesTable(IHtmlDocument document)
|
||||
{
|
||||
foreach (var table in document.QuerySelectorAll("table"))
|
||||
{
|
||||
if (table is not IHtmlTableElement tableElement)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var header = tableElement.Rows.FirstOrDefault();
|
||||
if (header is null || header.Cells.Length < 4)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var firstHeader = NormalizeWhitespace(header.Cells[0].TextContent);
|
||||
var secondHeader = NormalizeWhitespace(header.Cells[1].TextContent);
|
||||
var thirdHeader = NormalizeWhitespace(header.Cells[2].TextContent);
|
||||
if (string.Equals(firstHeader, "Source Package", StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(secondHeader, "Release", StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(thirdHeader, "Version", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return tableElement;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string NormalizeRelease(string release)
|
||||
{
|
||||
var trimmed = release.Trim();
|
||||
var parenthesisIndex = trimmed.IndexOf('(');
|
||||
if (parenthesisIndex > 0)
|
||||
{
|
||||
trimmed = trimmed[..parenthesisIndex].Trim();
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private static string ExtractPackageName(string value)
|
||||
{
|
||||
var trimmed = value.Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
|
||||
if (trimmed.EndsWith(")", StringComparison.Ordinal) && trimmed.Contains('('))
|
||||
{
|
||||
trimmed = trimmed[..trimmed.IndexOf('(')];
|
||||
}
|
||||
|
||||
return trimmed.Trim();
|
||||
}
|
||||
|
||||
private static string NormalizeWhitespace(string value)
|
||||
=> string.IsNullOrWhiteSpace(value)
|
||||
? string.Empty
|
||||
: string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
|
||||
|
||||
private sealed class PackageAccumulator
|
||||
{
|
||||
private readonly DateTimeOffset _published;
|
||||
|
||||
public PackageAccumulator(string package, string release, DateTimeOffset published)
|
||||
{
|
||||
Package = package;
|
||||
Release = release;
|
||||
_published = published;
|
||||
Status = "unknown";
|
||||
}
|
||||
|
||||
public string Package { get; }
|
||||
|
||||
public string Release { get; }
|
||||
|
||||
public string Status { get; private set; }
|
||||
|
||||
public string? IntroducedVersion { get; private set; }
|
||||
|
||||
public string? FixedVersion { get; private set; }
|
||||
|
||||
public string? LastAffectedVersion { get; private set; }
|
||||
|
||||
public bool ShouldEmit =>
|
||||
!string.Equals(Status, "not_affected", StringComparison.OrdinalIgnoreCase)
|
||||
|| IntroducedVersion is not null
|
||||
|| FixedVersion is not null;
|
||||
|
||||
public void Apply(string statusRaw, string versionRaw)
|
||||
{
|
||||
var status = statusRaw.ToLowerInvariant();
|
||||
var version = string.IsNullOrWhiteSpace(versionRaw) ? null : versionRaw.Trim();
|
||||
|
||||
if (status.Contains("fixed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
FixedVersion = version;
|
||||
if (!string.Equals(Status, "open", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Status = "resolved";
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (status.Contains("vulnerable", StringComparison.OrdinalIgnoreCase)
|
||||
|| status.Contains("open", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
IntroducedVersion ??= version;
|
||||
if (!string.Equals(Status, "resolved", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Status = "open";
|
||||
}
|
||||
|
||||
LastAffectedVersion = null;
|
||||
return;
|
||||
}
|
||||
|
||||
if (status.Contains("not affected", StringComparison.OrdinalIgnoreCase)
|
||||
|| status.Contains("not vulnerable", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Status = "not_affected";
|
||||
IntroducedVersion = null;
|
||||
FixedVersion = null;
|
||||
LastAffectedVersion = null;
|
||||
return;
|
||||
}
|
||||
|
||||
if (status.Contains("end-of-life", StringComparison.OrdinalIgnoreCase) || status.Contains("end of life", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Status = "end_of_life";
|
||||
return;
|
||||
}
|
||||
|
||||
Status = statusRaw;
|
||||
}
|
||||
|
||||
public DebianPackageStateDto ToDto()
|
||||
=> new(
|
||||
Package: Package,
|
||||
Release: Release,
|
||||
Status: Status,
|
||||
IntroducedVersion: IntroducedVersion,
|
||||
FixedVersion: FixedVersion,
|
||||
LastAffectedVersion: LastAffectedVersion,
|
||||
Published: _published);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal sealed record DebianListEntry(
|
||||
string AdvisoryId,
|
||||
DateTimeOffset Published,
|
||||
string Title,
|
||||
string SourcePackage,
|
||||
IReadOnlyList<string> CveIds);
|
||||
@@ -0,0 +1,107 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal static class DebianListParser
|
||||
{
|
||||
private static readonly Regex HeaderRegex = new("^\\[(?<date>[^\\]]+)\\]\\s+(?<id>DSA-\\d{4,}-\\d+)\\s+(?<title>.+)$", RegexOptions.Compiled);
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyList<DebianListEntry> Parse(string? content)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return Array.Empty<DebianListEntry>();
|
||||
}
|
||||
|
||||
var entries = new List<DebianListEntry>();
|
||||
var currentCves = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
DateTimeOffset currentDate = default;
|
||||
string? currentId = null;
|
||||
string? currentTitle = null;
|
||||
string? currentPackage = null;
|
||||
|
||||
foreach (var rawLine in content.Split('\n'))
|
||||
{
|
||||
var line = rawLine.TrimEnd('\r');
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line[0] == '[')
|
||||
{
|
||||
if (currentId is not null && currentTitle is not null && currentPackage is not null)
|
||||
{
|
||||
entries.Add(new DebianListEntry(
|
||||
currentId,
|
||||
currentDate,
|
||||
currentTitle,
|
||||
currentPackage,
|
||||
currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves)));
|
||||
}
|
||||
|
||||
currentCves.Clear();
|
||||
currentId = null;
|
||||
currentTitle = null;
|
||||
currentPackage = null;
|
||||
|
||||
var match = HeaderRegex.Match(line);
|
||||
if (!match.Success)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParseExact(
|
||||
match.Groups["date"].Value,
|
||||
new[] { "dd MMM yyyy", "d MMM yyyy" },
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
|
||||
out currentDate))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
currentId = match.Groups["id"].Value.Trim();
|
||||
currentTitle = match.Groups["title"].Value.Trim();
|
||||
|
||||
var separatorIndex = currentTitle.IndexOf(" - ", StringComparison.Ordinal);
|
||||
currentPackage = separatorIndex > 0
|
||||
? currentTitle[..separatorIndex].Trim()
|
||||
: currentTitle.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(currentPackage))
|
||||
{
|
||||
currentPackage = currentId;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line[0] == '{')
|
||||
{
|
||||
foreach (Match match in CveRegex.Matches(line))
|
||||
{
|
||||
if (match.Success && !string.IsNullOrWhiteSpace(match.Value))
|
||||
{
|
||||
currentCves.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (currentId is not null && currentTitle is not null && currentPackage is not null)
|
||||
{
|
||||
entries.Add(new DebianListEntry(
|
||||
currentId,
|
||||
currentDate,
|
||||
currentTitle,
|
||||
currentPackage,
|
||||
currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves)));
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Normalization.Distro;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian.Internal;
|
||||
|
||||
internal static class DebianMapper
|
||||
{
|
||||
public static Advisory Map(
|
||||
DebianAdvisoryDto dto,
|
||||
DocumentRecord document,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var affectedPackages = BuildAffectedPackages(dto, recordedAt);
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(
|
||||
DebianConnectorPlugin.SourceName,
|
||||
"document",
|
||||
document.Uri,
|
||||
document.FetchedAt.ToUniversalTime());
|
||||
|
||||
var mappingProvenance = new AdvisoryProvenance(
|
||||
DebianConnectorPlugin.SourceName,
|
||||
"mapping",
|
||||
dto.AdvisoryId,
|
||||
recordedAt);
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.AdvisoryId,
|
||||
title: dto.Title ?? dto.AdvisoryId,
|
||||
summary: dto.Description,
|
||||
language: "en",
|
||||
published: dto.Packages.Select(p => p.Published).Where(p => p.HasValue).Select(p => p!.Value).Cast<DateTimeOffset?>().DefaultIfEmpty(null).Min(),
|
||||
modified: recordedAt,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mappingProvenance });
|
||||
}
|
||||
|
||||
private static string[] BuildAliases(DebianAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (!string.IsNullOrWhiteSpace(dto.AdvisoryId))
|
||||
{
|
||||
aliases.Add(dto.AdvisoryId.Trim());
|
||||
}
|
||||
|
||||
foreach (var cve in dto.CveIds ?? Array.Empty<string>())
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
return aliases.OrderBy(a => a, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
private static AdvisoryReference[] BuildReferences(DebianAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.References is null || dto.References.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>();
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
DebianConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt);
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
NormalizeReferenceKind(reference.Kind),
|
||||
reference.Kind,
|
||||
reference.Title,
|
||||
provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// Ignore malformed URLs while keeping the rest of the advisory intact.
|
||||
}
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<AdvisoryReference>()
|
||||
: references
|
||||
.OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceKind(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"advisory" or "dsa" => "advisory",
|
||||
"cve" => "cve",
|
||||
"patch" => "patch",
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static AdvisoryProvenance BuildPackageProvenance(DebianPackageStateDto package, DateTimeOffset recordedAt)
|
||||
=> new(DebianConnectorPlugin.SourceName, "affected", $"{package.Package}:{package.Release}", recordedAt);
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(DebianAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Packages is null || dto.Packages.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Packages.Count);
|
||||
foreach (var package in dto.Packages)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(package.Package))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new[] { BuildPackageProvenance(package, recordedAt) };
|
||||
var ranges = BuildVersionRanges(package, recordedAt);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Deb,
|
||||
identifier: package.Package.Trim(),
|
||||
platform: package.Release,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(DebianPackageStateDto package, DateTimeOffset recordedAt)
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
DebianConnectorPlugin.SourceName,
|
||||
"range",
|
||||
$"{package.Package}:{package.Release}",
|
||||
recordedAt);
|
||||
|
||||
var introduced = package.IntroducedVersion;
|
||||
var fixedVersion = package.FixedVersion;
|
||||
var lastAffected = package.LastAffectedVersion;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(introduced) && string.IsNullOrWhiteSpace(fixedVersion) && string.IsNullOrWhiteSpace(lastAffected))
|
||||
{
|
||||
return Array.Empty<AffectedVersionRange>();
|
||||
}
|
||||
|
||||
var extensions = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["debian.release"] = package.Release,
|
||||
["debian.status"] = package.Status
|
||||
};
|
||||
|
||||
AddExtension(extensions, "debian.introduced", introduced);
|
||||
AddExtension(extensions, "debian.fixed", fixedVersion);
|
||||
AddExtension(extensions, "debian.lastAffected", lastAffected);
|
||||
|
||||
var primitives = BuildEvrPrimitives(introduced, fixedVersion, lastAffected);
|
||||
return new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: "evr",
|
||||
introducedVersion: introduced,
|
||||
fixedVersion: fixedVersion,
|
||||
lastAffectedVersion: lastAffected,
|
||||
rangeExpression: BuildRangeExpression(introduced, fixedVersion, lastAffected),
|
||||
provenance: provenance,
|
||||
primitives: primitives is null && extensions.Count == 0
|
||||
? null
|
||||
: new RangePrimitives(
|
||||
SemVer: null,
|
||||
Nevra: null,
|
||||
Evr: primitives,
|
||||
VendorExtensions: extensions.Count == 0 ? null : extensions))
|
||||
};
|
||||
}
|
||||
|
||||
private static EvrPrimitive? BuildEvrPrimitives(string? introduced, string? fixedVersion, string? lastAffected)
|
||||
{
|
||||
var introducedComponent = ParseEvr(introduced);
|
||||
var fixedComponent = ParseEvr(fixedVersion);
|
||||
var lastAffectedComponent = ParseEvr(lastAffected);
|
||||
|
||||
if (introducedComponent is null && fixedComponent is null && lastAffectedComponent is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new EvrPrimitive(introducedComponent, fixedComponent, lastAffectedComponent);
|
||||
}
|
||||
|
||||
private static EvrComponent? ParseEvr(string? value)
|
||||
{
|
||||
if (!DebianEvr.TryParse(value, out var evr) || evr is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new EvrComponent(
|
||||
evr.Epoch,
|
||||
evr.Version,
|
||||
evr.Revision.Length == 0 ? null : evr.Revision);
|
||||
}
|
||||
|
||||
private static string? BuildRangeExpression(string? introduced, string? fixedVersion, string? lastAffected)
|
||||
{
|
||||
var parts = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(introduced))
|
||||
{
|
||||
parts.Add($"introduced:{introduced.Trim()}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(fixedVersion))
|
||||
{
|
||||
parts.Add($"fixed:{fixedVersion.Trim()}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lastAffected))
|
||||
{
|
||||
parts.Add($"last:{lastAffected.Trim()}");
|
||||
}
|
||||
|
||||
return parts.Count == 0 ? null : string.Join(" ", parts);
|
||||
}
|
||||
|
||||
private static void AddExtension(IDictionary<string, string> extensions, string key, string? value)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
extensions[key] = value.Trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
46
src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs
Normal file
46
src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs
Normal file
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Distro.Debian;
|
||||
|
||||
internal static class DebianJobKinds
|
||||
{
|
||||
public const string Fetch = "source:debian:fetch";
|
||||
public const string Parse = "source:debian:parse";
|
||||
public const string Map = "source:debian:map";
|
||||
}
|
||||
|
||||
internal sealed class DebianFetchJob : IJob
|
||||
{
|
||||
private readonly DebianConnector _connector;
|
||||
|
||||
public DebianFetchJob(DebianConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class DebianParseJob : IJob
|
||||
{
|
||||
private readonly DebianConnector _connector;
|
||||
|
||||
public DebianParseJob(DebianConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class DebianMapJob : IJob
|
||||
{
|
||||
private readonly DebianConnector _connector;
|
||||
|
||||
public DebianMapJob(DebianConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user