Resolve Concelier/Excititor merge conflicts
This commit is contained in:
master
40
src/StellaOps.Concelier.Connector.CertBund/AGENTS.md
Normal file
40
src/StellaOps.Concelier.Connector.CertBund/AGENTS.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Deliver a connector for Germany’s CERT-Bund advisories so Concelier can ingest, normalise, and enrich BSI alerts alongside other national feeds.
|
||||
|
||||
## Scope
|
||||
- Identify the authoritative CERT-Bund advisory feed(s) (RSS/Atom, JSON, CSV, or HTML).
|
||||
- Implement fetch/cursor logic with proper windowing, dedupe, and failure backoff.
|
||||
- Parse advisory detail pages for summary, affected products/vendors, mitigation, and references.
|
||||
- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and provenance/range primitives.
|
||||
- Provide deterministic fixtures and regression tests.
|
||||
|
||||
## Participants
|
||||
- `Source.Common` (HTTP/fetch utilities, DTO storage).
|
||||
- `Storage.Mongo` (raw/document/DTO/advisory stores, source state).
|
||||
- `Concelier.Models` (canonical data model).
|
||||
- `Concelier.Testing` (integration harness, snapshot utilities).
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `certbund:fetch`, `certbund:parse`, `certbund:map`.
|
||||
- Persist upstream metadata (ETag/Last-Modified) if provided.
|
||||
- Alias set should include CERT-Bund ID and referenced CVE entries.
|
||||
|
||||
## In/Out of scope
|
||||
In scope:
|
||||
- End-to-end connector implementation with deterministic tests and range primitive coverage.
|
||||
- Baseline logging/metrics for pipeline observability.
|
||||
|
||||
Out of scope:
|
||||
- Non-advisory CERT-Bund digests or newsletters.
|
||||
- Downstream exporter changes.
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch attempts, item counts, and mapping metrics.
|
||||
- Sanitize HTML thoroughly before persistence.
|
||||
- Handle transient failures gracefully with exponential backoff and failure records in source state.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.CertBund.Tests` covering fetch/parse/map with canned fixtures.
|
||||
- Snapshot canonical advisories; support regeneration via environment flag.
|
||||
- Ensure deterministic ordering, casing, and timestamps.
|
||||
435
src/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs
Normal file
435
src/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs
Normal file
@@ -0,0 +1,435 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.CertBund.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertBund.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertBund;
|
||||
|
||||
public sealed class CertBundConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly CertBundFeedClient _feedClient;
|
||||
private readonly CertBundDetailParser _detailParser;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly CertBundOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly CertBundDiagnostics _diagnostics;
|
||||
private readonly ILogger<CertBundConnector> _logger;
|
||||
|
||||
public CertBundConnector(
|
||||
CertBundFeedClient feedClient,
|
||||
CertBundDetailParser detailParser,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<CertBundOptions> options,
|
||||
CertBundDiagnostics diagnostics,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<CertBundConnector> logger)
|
||||
{
|
||||
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
|
||||
_detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => CertBundConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
IReadOnlyList<CertBundFeedItem> feedItems;
|
||||
|
||||
_diagnostics.FeedFetchAttempt();
|
||||
try
|
||||
{
|
||||
feedItems = await _feedClient.LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
_diagnostics.FeedFetchSuccess(feedItems.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund feed fetch failed");
|
||||
_diagnostics.FeedFetchFailure();
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
var coverageDays = CalculateCoverageDays(feedItems, now);
|
||||
_diagnostics.RecordFeedCoverage(coverageDays);
|
||||
|
||||
if (feedItems.Count == 0)
|
||||
{
|
||||
await UpdateCursorAsync(cursor.WithLastFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var knownAdvisories = new HashSet<string>(cursor.KnownAdvisories, StringComparer.OrdinalIgnoreCase);
|
||||
var processed = 0;
|
||||
var alreadyKnown = 0;
|
||||
var notModified = 0;
|
||||
var detailFailures = 0;
|
||||
var truncated = false;
|
||||
var latestPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
|
||||
|
||||
foreach (var item in feedItems.OrderByDescending(static i => i.Published))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (knownAdvisories.Contains(item.AdvisoryId))
|
||||
{
|
||||
alreadyKnown++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (processed >= _options.MaxAdvisoriesPerFetch)
|
||||
{
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_diagnostics.DetailFetchAttempt();
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
var request = new SourceFetchRequest(CertBundOptions.HttpClientName, SourceName, item.DetailUri)
|
||||
{
|
||||
AcceptHeaders = new[] { "application/json", "text/json" },
|
||||
Metadata = CertBundDocumentMetadata.CreateMetadata(item),
|
||||
ETag = existing?.Etag,
|
||||
LastModified = existing?.LastModified,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
_diagnostics.DetailFetchNotModified();
|
||||
notModified++;
|
||||
knownAdvisories.Add(item.AdvisoryId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
_diagnostics.DetailFetchFailure("skipped");
|
||||
detailFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.DetailFetchSuccess();
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
pendingMappings.Remove(result.Document.Id);
|
||||
knownAdvisories.Add(item.AdvisoryId);
|
||||
processed++;
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund detail fetch failed for {AdvisoryId}", item.AdvisoryId);
|
||||
_diagnostics.DetailFetchFailure("exception");
|
||||
detailFailures++;
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (item.Published > latestPublished)
|
||||
{
|
||||
latestPublished = item.Published;
|
||||
}
|
||||
}
|
||||
|
||||
_diagnostics.DetailFetchEnqueued(processed);
|
||||
|
||||
if (feedItems.Count > 0 || processed > 0 || detailFailures > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CERT-Bund fetch cycle: feed items {FeedItems}, enqueued {Enqueued}, already known {Known}, not modified {NotModified}, detail failures {DetailFailures}, pending documents {PendingDocuments}, pending mappings {PendingMappings}, truncated {Truncated}, coverageDays={CoverageDays}",
|
||||
feedItems.Count,
|
||||
processed,
|
||||
alreadyKnown,
|
||||
notModified,
|
||||
detailFailures,
|
||||
pendingDocuments.Count,
|
||||
pendingMappings.Count,
|
||||
truncated,
|
||||
coverageDays ?? double.NaN);
|
||||
}
|
||||
|
||||
var trimmedKnown = knownAdvisories.Count > _options.MaxKnownAdvisories
|
||||
? knownAdvisories.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxKnownAdvisories)
|
||||
.ToArray()
|
||||
: knownAdvisories.ToArray();
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownAdvisories(trimmedKnown)
|
||||
.WithLastPublished(latestPublished == DateTimeOffset.MinValue ? cursor.LastPublished : latestPublished)
|
||||
.WithLastFetch(now);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var parsedCount = 0;
|
||||
var failedCount = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.ParseFailure("missing_payload");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund unable to download document {DocumentId}", document.Id);
|
||||
_diagnostics.ParseFailure("download_failed");
|
||||
throw;
|
||||
}
|
||||
|
||||
CertBundAdvisoryDto dto;
|
||||
try
|
||||
{
|
||||
dto = _detailParser.Parse(new Uri(document.Uri), new Uri(document.Metadata?["certbund.portalUri"] ?? document.Uri), payload);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund failed to parse advisory detail {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.ParseFailure("parse_error");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.ParseSuccess(dto.Products.Count, dto.CveIds.Count);
|
||||
parsedCount++;
|
||||
|
||||
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", bson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Add(document.Id);
|
||||
}
|
||||
|
||||
if (cursor.PendingDocuments.Count > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CERT-Bund parse cycle: parsed {Parsed}, failures {Failures}, remaining documents {RemainingDocuments}, pending mappings {PendingMappings}",
|
||||
parsedCount,
|
||||
failedCount,
|
||||
remainingDocuments.Count,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var mappedCount = 0;
|
||||
var failedCount = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("missing_dto");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
CertBundAdvisoryDto? dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<CertBundAdvisoryDto>(dtoRecord.Payload.ToJson(), SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund failed to deserialize DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("deserialize_failed");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("null_dto");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = CertBundMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapSuccess(advisory.AffectedPackages.Length, advisory.Aliases.Length);
|
||||
mappedCount++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund mapping failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("exception");
|
||||
failedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (cursor.PendingMappings.Count > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CERT-Bund map cycle: mapped {Mapped}, failures {Failures}, remaining pending mappings {PendingMappings}",
|
||||
mappedCount,
|
||||
failedCount,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static double? CalculateCoverageDays(IReadOnlyList<CertBundFeedItem> items, DateTimeOffset fetchedAt)
|
||||
{
|
||||
if (items is null || items.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var oldest = items.Min(static item => item.Published);
|
||||
if (oldest == DateTimeOffset.MinValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var span = fetchedAt - oldest;
|
||||
return span >= TimeSpan.Zero ? span.TotalDays : null;
|
||||
}
|
||||
|
||||
private async Task<CertBundCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? CertBundCursor.Empty : CertBundCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(CertBundCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertBund;
|
||||
|
||||
public sealed class CertBundConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-bund";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CertBundConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CertBundConnector>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.CertBund.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertBund;
|
||||
|
||||
public sealed class CertBundDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cert-bund";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertBundConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertBundFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertBundJobKinds.Fetch, typeof(CertBundFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertBund.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertBund.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertBund;
|
||||
|
||||
public static class CertBundServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertBundConnector(this IServiceCollection services, Action<CertBundOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertBundOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertBundOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertBundOptions>>().Value;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.CertBund/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
|
||||
clientOptions.AllowedHosts.Add(options.DetailApiUri.Host);
|
||||
clientOptions.AllowedHosts.Add(options.PortalBootstrapUri.Host);
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.All;
|
||||
handler.UseCookies = true;
|
||||
handler.CookieContainer = new System.Net.CookieContainer();
|
||||
};
|
||||
});
|
||||
|
||||
services.TryAddSingleton<HtmlContentSanitizer>();
|
||||
services.TryAddSingleton<CertBundDiagnostics>();
|
||||
services.TryAddSingleton<CertBundFeedClient>();
|
||||
services.TryAddSingleton<CertBundDetailParser>();
|
||||
services.AddTransient<CertBundConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
22
src/StellaOps.Concelier.Connector.CertBund/Jobs.cs
Normal file
22
src/StellaOps.Concelier.Connector.CertBund/Jobs.cs
Normal file
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertBund;
|
||||
|
||||
internal static class CertBundJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-bund:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CertBundFetchJob : IJob
|
||||
{
|
||||
private readonly CertBundConnector _connector;
|
||||
|
||||
public CertBundFetchJob(CertBundConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
39
src/StellaOps.Concelier.Connector.CertBund/README.md
Normal file
39
src/StellaOps.Concelier.Connector.CertBund/README.md
Normal file
@@ -0,0 +1,39 @@
|
||||
# CERT-Bund Security Advisories – Connector Notes
|
||||
|
||||
## Publication endpoints
|
||||
- **RSS feed (latest 250 advisories)** – `https://wid.cert-bund.de/content/public/securityAdvisory/rss`. The feed refreshes quickly; the current window spans roughly 6 days of activity, so fetch jobs must run frequently to avoid churn.
|
||||
- **Portal bootstrap** – `https://wid.cert-bund.de/portal/` is hit once per process start to prime the session (`client_config` cookie) before any API calls.
|
||||
- **Detail API** – `https://wid.cert-bund.de/portal/api/securityadvisory?name=<ID>`. The connector reuses the bootstrapped `SocketsHttpHandler` so cookies and headers match the Angular SPA. Manual reproduction requires the same cookie container; otherwise the endpoint responds with the shell HTML document.
|
||||
|
||||
## Telemetry
|
||||
The OpenTelemetry meter is `StellaOps.Concelier.Connector.CertBund`. Key instruments:
|
||||
|
||||
| Metric | Type | Notes |
|
||||
| --- | --- | --- |
|
||||
| `certbund.feed.fetch.attempts` / `.success` / `.failures` | counter | Feed poll lifecycle. |
|
||||
| `certbund.feed.items.count` | histogram | Items returned per RSS fetch. |
|
||||
| `certbund.feed.enqueued.count` | histogram | Detail documents queued per cycle (post-dedupe, before truncation). |
|
||||
| `certbund.feed.coverage.days` | histogram | Rolling window (fetch time − oldest published entry). Useful to alert when feed depth contracts. |
|
||||
| `certbund.detail.fetch.*` | counter | Attempts, successes, HTTP 304, and failure counts; failures are tagged by reason (`skipped`, `exception`). |
|
||||
| `certbund.parse.success` / `.failures` | counter | Parsing outcomes; histograms capture product and CVE counts. |
|
||||
| `certbund.map.success` / `.failures` | counter | Canonical mapping results; histograms capture affected-package and alias volume. |
|
||||
|
||||
Dashboards should chart coverage days and enqueued counts alongside fetch failures: sharp drops indicate the upstream window tightened or parsing stalled.
|
||||
|
||||
## Logging signals
|
||||
- `CERT-Bund fetch cycle: feed items …` summarises each RSS run (enqueued, already-known, HTTP 304, failures, coverage window).
|
||||
- Parse and map stages log corresponding counts when work remains in the cursor.
|
||||
- Errors include advisory/document identifiers to simplify replays.
|
||||
|
||||
## Historical coverage
|
||||
- RSS contains the newest **250** items (≈6 days at the current publication rate). The connector prunes the “known advisory” set to 512 IDs to avoid unbounded memory but retains enough headroom for short-term replay.
|
||||
- Older advisories remain accessible through the same detail API (`WID-SEC-<year>-<sequence>` identifiers). For deep backfills run a scripted sweep that queues historical IDs in descending order; the connector will persist any payloads that still resolve. Document these batches under source state comments so Merge/Docs can track provenance.
|
||||
|
||||
## Locale & translation stance
|
||||
- CERT-Bund publishes advisory titles and summaries **only in German** (language tag `de`). The connector preserves original casing/content and sets `Advisory.Language = "de"`.
|
||||
- Operator guidance:
|
||||
1. Front-line analysts consuming Concelier data should maintain German literacy or rely on approved machine-translation pipelines.
|
||||
2. When mirroring advisories into English dashboards, store translations outside the canonical advisory payload to keep determinism. Suggested approach: create an auxiliary collection keyed by advisory ID with timestamped translated snippets.
|
||||
3. Offline Kit bundles must document that CERT-Bund content is untranslated to avoid surprise during audits.
|
||||
|
||||
The Docs guild will surface the translation policy (retain German source, optionally layer operator-provided translations) in the broader i18n section; this README is the connector-level reference.
|
||||
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
13
src/StellaOps.Concelier.Connector.CertBund/TASKS.md
Normal file
13
src/StellaOps.Concelier.Connector.CertBund/TASKS.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-CERTBUND-02-001 Research CERT-Bund advisory endpoints|BE-Conn-CERTBUND|Research|**DONE (2025-10-11)** – Confirmed public RSS at `https://wid.cert-bund.de/content/public/securityAdvisory/rss` (HTTP 200 w/out cookies), 250-item window, German titles/categories, and detail links pointing to Angular SPA. Captured header profile (no cache hints) and logged open item to discover the JSON API used by `portal` frontend.|
|
||||
|FEEDCONN-CERTBUND-02-002 Fetch job & state persistence|BE-Conn-CERTBUND|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `CertBundConnector.FetchAsync` consumes RSS via session-bootstrapped client, stores per-advisory JSON documents with metadata + SHA, throttles detail requests, and maintains cursor state (pending docs/mappings, known advisory IDs, last published).|
|
||||
|FEEDCONN-CERTBUND-02-003 Parser/DTO implementation|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Detail JSON piped through `CertBundDetailParser` (raw DOM sanitised to HTML), capturing severity, CVEs, product list, and references into DTO records (`cert-bund.detail.v1`).|
|
||||
|FEEDCONN-CERTBUND-02-004 Canonical mapping & range primitives|BE-Conn-CERTBUND|Models|**DONE (2025-10-14)** – `CertBundMapper` emits canonical advisories (aliases, references, vendor package ranges, provenance) with severity normalisation and deterministic ordering.|
|
||||
|FEEDCONN-CERTBUND-02-005 Regression fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added `StellaOps.Concelier.Connector.CertBund.Tests` covering fetch→parse→map against canned RSS/JSON fixtures; integration harness uses Mongo2Go + canned HTTP handler; fixtures regenerate via `UPDATE_CERTBUND_FIXTURES=1`.|
|
||||
|FEEDCONN-CERTBUND-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CertBundDiagnostics` (meter `StellaOps.Concelier.Connector.CertBund`) with fetch/parse/map counters + histograms, recorded coverage days, wired stage summary logs, and published the ops runbook (`docs/ops/concelier-certbund-operations.md`).|
|
||||
|FEEDCONN-CERTBUND-02-007 Feed history & locale assessment|BE-Conn-CERTBUND|Research|**DONE (2025-10-15)** – Measured RSS retention (~6 days/≈250 items), captured connector-driven backfill guidance in the runbook, and aligned locale guidance (preserve `language=de`, Docs glossary follow-up). **Next:** coordinate with Tools to land the state-seeding helper so scripted backfills replace manual Mongo tweaks.|
|
||||
|FEEDCONN-CERTBUND-02-008 Session bootstrap & cookie strategy|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Feed client primes the portal session (cookie container via `SocketsHttpHandler`), shares cookies across detail requests, and documents bootstrap behaviour in options (`PortalBootstrapUri`).|
|
||||
|FEEDCONN-CERTBUND-02-009 Offline Kit export packaging|BE-Conn-CERTBUND, Docs|Offline Kit|**DONE (2025-10-17)** – Added `tools/certbund_offline_snapshot.py` to capture search/export JSON, emit deterministic manifests + SHA files, and refreshed docs (`docs/ops/concelier-certbund-operations.md`, `docs/24_OFFLINE_KIT.md`) with offline-kit instructions and manifest layout guidance. Seed data README/ignore rules cover local snapshot hygiene.|
|
||||
|FEEDCONN-CERTBUND-02-010 Normalized range translator|BE-Conn-CERTBUND|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-22)** – Translate `product.Versions` phrases (e.g., `2023.1 bis 2024.2`, `alle`) into comparator strings for `SemVerRangeRuleBuilder`, emit `NormalizedVersions` with `certbund:{advisoryId}:{vendor}` provenance, and extend tests/README with localisation notes.|
|
||||
Reference in New Issue
Block a user