feat: Initialize Zastava Webhook service with TLS and Authority authentication
- Added Program.cs to set up the web application with Serilog for logging, health check endpoints, and a placeholder admission endpoint. - Configured Kestrel server to use TLS 1.3 and handle client certificates appropriately. - Created StellaOps.Zastava.Webhook.csproj with necessary dependencies including Serilog and Polly. - Documented tasks in TASKS.md for the Zastava Webhook project, outlining current work and exit criteria for each task.
This commit is contained in:
master
@@ -0,0 +1,159 @@
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.State;
|
||||
|
||||
/// <summary>
|
||||
/// Describes a raw upstream document that should be persisted for a connector during seeding.
|
||||
/// </summary>
|
||||
public sealed record SourceStateSeedDocument
|
||||
{
|
||||
/// <summary>
|
||||
/// Absolute source URI. Must match the connector's upstream document identifier.
|
||||
/// </summary>
|
||||
public string Uri { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Raw document payload. Required when creating or replacing a document.
|
||||
/// </summary>
|
||||
public byte[] Content { get; init; } = Array.Empty<byte>();
|
||||
|
||||
/// <summary>
|
||||
/// Optional explicit document identifier. When provided it overrides auto-generated IDs.
|
||||
/// </summary>
|
||||
public Guid? DocumentId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// MIME type for the document payload.
|
||||
/// </summary>
|
||||
public string? ContentType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Status assigned to the document. Defaults to <see cref="DocumentStatuses.PendingParse"/>.
|
||||
/// </summary>
|
||||
public string Status { get; init; } = DocumentStatuses.PendingParse;
|
||||
|
||||
/// <summary>
|
||||
/// Optional HTTP-style headers persisted alongside the raw document.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string>? Headers { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Source metadata (connector specific) persisted alongside the raw document.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Upstream ETag value, if available.
|
||||
/// </summary>
|
||||
public string? Etag { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Upstream last-modified timestamp, if available.
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastModified { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional document expiration. When set a TTL will purge the raw payload after the configured retention.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Fetch timestamp stamped onto the document. Defaults to the seed completion timestamp.
|
||||
/// </summary>
|
||||
public DateTimeOffset? FetchedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When true, the document ID will be appended to the connector cursor's <c>pendingDocuments</c> set.
|
||||
/// </summary>
|
||||
public bool AddToPendingDocuments { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// When true, the document ID will be appended to the connector cursor's <c>pendingMappings</c> set.
|
||||
/// </summary>
|
||||
public bool AddToPendingMappings { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional identifiers that should be recorded on the cursor to avoid duplicate ingestion.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? KnownIdentifiers { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cursor updates that should accompany seeded documents.
|
||||
/// </summary>
|
||||
public sealed record SourceStateSeedCursor
|
||||
{
|
||||
/// <summary>
|
||||
/// Optional <c>pendingDocuments</c> additions expressed as document IDs.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<Guid>? PendingDocuments { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional <c>pendingMappings</c> additions expressed as document IDs.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<Guid>? PendingMappings { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional known advisory identifiers to merge with the cursor.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Upstream window watermark tracked by connectors that rely on last-modified cursors.
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastModifiedCursor { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional fetch timestamp used by connectors that track the last polling instant.
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastFetchAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional cursor fields (string values) to merge.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string>? Additional { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeding specification describing the source, documents, and cursor edits to apply.
|
||||
/// </summary>
|
||||
public sealed record SourceStateSeedSpecification
|
||||
{
|
||||
/// <summary>
|
||||
/// Source/connector name (e.g. <c>vndr.msrc</c>).
|
||||
/// </summary>
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Documents that should be inserted or replaced before the cursor update.
|
||||
/// </summary>
|
||||
public IReadOnlyList<SourceStateSeedDocument> Documents { get; init; } = Array.Empty<SourceStateSeedDocument>();
|
||||
|
||||
/// <summary>
|
||||
/// Cursor adjustments applied after documents are persisted.
|
||||
/// </summary>
|
||||
public SourceStateSeedCursor? Cursor { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Connector-level known advisory identifiers to merge into the cursor.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? KnownAdvisories { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional completion timestamp. Defaults to the processor's time provider.
|
||||
/// </summary>
|
||||
public DateTimeOffset? CompletedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result returned after seeding completes.
|
||||
/// </summary>
|
||||
public sealed record SourceStateSeedResult(
|
||||
int DocumentsProcessed,
|
||||
int PendingDocumentsAdded,
|
||||
int PendingMappingsAdded,
|
||||
IReadOnlyCollection<Guid> DocumentIds,
|
||||
IReadOnlyCollection<Guid> PendingDocumentIds,
|
||||
IReadOnlyCollection<Guid> PendingMappingIds,
|
||||
IReadOnlyCollection<string> KnownAdvisoriesAdded,
|
||||
DateTimeOffset CompletedAt);
|
||||
@@ -0,0 +1,329 @@
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Common.State;
|
||||
|
||||
/// <summary>
|
||||
/// Persists raw documents and cursor state for connectors that require manual seeding.
|
||||
/// </summary>
|
||||
public sealed class SourceStateSeedProcessor
|
||||
{
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<SourceStateSeedProcessor> _logger;
|
||||
|
||||
public SourceStateSeedProcessor(
|
||||
IDocumentStore documentStore,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
ISourceStateRepository stateRepository,
|
||||
TimeProvider? timeProvider = null,
|
||||
ILogger<SourceStateSeedProcessor>? logger = null)
|
||||
{
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? NullLogger<SourceStateSeedProcessor>.Instance;
|
||||
}
|
||||
|
||||
public async Task<SourceStateSeedResult> ProcessAsync(SourceStateSeedSpecification specification, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(specification);
|
||||
ArgumentException.ThrowIfNullOrEmpty(specification.Source);
|
||||
|
||||
var completedAt = specification.CompletedAt ?? _timeProvider.GetUtcNow();
|
||||
var documentIds = new List<Guid>();
|
||||
var pendingDocumentIds = new HashSet<Guid>();
|
||||
var pendingMappingIds = new HashSet<Guid>();
|
||||
var knownAdvisories = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
AppendRange(knownAdvisories, specification.KnownAdvisories);
|
||||
|
||||
if (specification.Cursor is { } cursorSeed)
|
||||
{
|
||||
AppendRange(pendingDocumentIds, cursorSeed.PendingDocuments);
|
||||
AppendRange(pendingMappingIds, cursorSeed.PendingMappings);
|
||||
AppendRange(knownAdvisories, cursorSeed.KnownAdvisories);
|
||||
}
|
||||
|
||||
foreach (var document in specification.Documents ?? Array.Empty<SourceStateSeedDocument>())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await ProcessDocumentAsync(specification.Source, document, completedAt, documentIds, pendingDocumentIds, pendingMappingIds, knownAdvisories, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var state = await _stateRepository.TryGetAsync(specification.Source, cancellationToken).ConfigureAwait(false);
|
||||
var cursor = state?.Cursor ?? new BsonDocument();
|
||||
|
||||
var newlyPendingDocuments = MergeGuidArray(cursor, "pendingDocuments", pendingDocumentIds);
|
||||
var newlyPendingMappings = MergeGuidArray(cursor, "pendingMappings", pendingMappingIds);
|
||||
var newlyKnownAdvisories = MergeStringArray(cursor, "knownAdvisories", knownAdvisories);
|
||||
|
||||
if (specification.Cursor is { } cursorSpec)
|
||||
{
|
||||
if (cursorSpec.LastModifiedCursor.HasValue)
|
||||
{
|
||||
cursor["lastModifiedCursor"] = cursorSpec.LastModifiedCursor.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (cursorSpec.LastFetchAt.HasValue)
|
||||
{
|
||||
cursor["lastFetchAt"] = cursorSpec.LastFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (cursorSpec.Additional is not null)
|
||||
{
|
||||
foreach (var kvp in cursorSpec.Additional)
|
||||
{
|
||||
cursor[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cursor["lastSeededAt"] = completedAt.UtcDateTime;
|
||||
await _stateRepository.UpdateCursorAsync(specification.Source, cursor, completedAt, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Seeded {Documents} document(s) for {Source}. pendingDocuments+= {PendingDocuments}, pendingMappings+= {PendingMappings}, knownAdvisories+= {KnownAdvisories}",
|
||||
documentIds.Count,
|
||||
specification.Source,
|
||||
newlyPendingDocuments.Count,
|
||||
newlyPendingMappings.Count,
|
||||
newlyKnownAdvisories.Count);
|
||||
|
||||
return new SourceStateSeedResult(
|
||||
DocumentsProcessed: documentIds.Count,
|
||||
PendingDocumentsAdded: newlyPendingDocuments.Count,
|
||||
PendingMappingsAdded: newlyPendingMappings.Count,
|
||||
DocumentIds: documentIds.AsReadOnly(),
|
||||
PendingDocumentIds: newlyPendingDocuments,
|
||||
PendingMappingIds: newlyPendingMappings,
|
||||
KnownAdvisoriesAdded: newlyKnownAdvisories,
|
||||
CompletedAt: completedAt);
|
||||
}
|
||||
|
||||
private async Task ProcessDocumentAsync(
|
||||
string source,
|
||||
SourceStateSeedDocument document,
|
||||
DateTimeOffset completedAt,
|
||||
List<Guid> documentIds,
|
||||
HashSet<Guid> pendingDocumentIds,
|
||||
HashSet<Guid> pendingMappingIds,
|
||||
HashSet<string> knownAdvisories,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (document is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(document));
|
||||
}
|
||||
|
||||
ArgumentException.ThrowIfNullOrEmpty(document.Uri);
|
||||
if (document.Content is not { Length: > 0 })
|
||||
{
|
||||
throw new InvalidOperationException($"Seed entry for '{document.Uri}' is missing content bytes.");
|
||||
}
|
||||
|
||||
var payload = new byte[document.Content.Length];
|
||||
Buffer.BlockCopy(document.Content, 0, payload, 0, document.Content.Length);
|
||||
|
||||
if (!document.Uri.Contains("://", StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogWarning("Seed document URI '{Uri}' does not appear to be absolute.", document.Uri);
|
||||
}
|
||||
|
||||
var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(source, document.Uri, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (existing?.GridFsId is { } oldGridId)
|
||||
{
|
||||
await _rawDocumentStorage.DeleteAsync(oldGridId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var gridId = await _rawDocumentStorage.UploadAsync(
|
||||
source,
|
||||
document.Uri,
|
||||
payload,
|
||||
document.ContentType,
|
||||
document.ExpiresAt,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var headers = CloneDictionary(document.Headers);
|
||||
if (!string.IsNullOrWhiteSpace(document.ContentType))
|
||||
{
|
||||
headers ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (!headers.ContainsKey("content-type"))
|
||||
{
|
||||
headers["content-type"] = document.ContentType!;
|
||||
}
|
||||
}
|
||||
|
||||
var metadata = CloneDictionary(document.Metadata);
|
||||
|
||||
var record = new DocumentRecord(
|
||||
document.DocumentId ?? existing?.Id ?? Guid.NewGuid(),
|
||||
source,
|
||||
document.Uri,
|
||||
document.FetchedAt ?? completedAt,
|
||||
sha256,
|
||||
string.IsNullOrWhiteSpace(document.Status) ? DocumentStatuses.PendingParse : document.Status,
|
||||
document.ContentType,
|
||||
headers,
|
||||
metadata,
|
||||
document.Etag,
|
||||
document.LastModified,
|
||||
gridId,
|
||||
document.ExpiresAt);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
documentIds.Add(upserted.Id);
|
||||
|
||||
if (document.AddToPendingDocuments)
|
||||
{
|
||||
pendingDocumentIds.Add(upserted.Id);
|
||||
}
|
||||
|
||||
if (document.AddToPendingMappings)
|
||||
{
|
||||
pendingMappingIds.Add(upserted.Id);
|
||||
}
|
||||
|
||||
AppendRange(knownAdvisories, document.KnownIdentifiers);
|
||||
}
|
||||
|
||||
private static Dictionary<string, string>? CloneDictionary(IReadOnlyDictionary<string, string>? values)
|
||||
{
|
||||
if (values is null || values.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Dictionary<string, string>(values, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> MergeGuidArray(BsonDocument cursor, string field, IReadOnlyCollection<Guid> additions)
|
||||
{
|
||||
if (additions.Count == 0)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray
|
||||
? existingArray.Select(AsGuid).Where(static g => g != Guid.Empty).ToHashSet()
|
||||
: new HashSet<Guid>();
|
||||
|
||||
var newlyAdded = new List<Guid>();
|
||||
foreach (var guid in additions)
|
||||
{
|
||||
if (guid == Guid.Empty)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing.Add(guid))
|
||||
{
|
||||
newlyAdded.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
if (existing.Count > 0)
|
||||
{
|
||||
cursor[field] = new BsonArray(existing
|
||||
.Select(static g => g.ToString("D"))
|
||||
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
return newlyAdded.AsReadOnly();
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> MergeStringArray(BsonDocument cursor, string field, IReadOnlyCollection<string> additions)
|
||||
{
|
||||
if (additions.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray
|
||||
? existingArray.Select(static v => v?.AsString ?? string.Empty)
|
||||
.Where(static s => !string.IsNullOrWhiteSpace(s))
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase)
|
||||
: new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var newlyAdded = new List<string>();
|
||||
foreach (var entry in additions)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(entry))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = entry.Trim();
|
||||
if (existing.Add(normalized))
|
||||
{
|
||||
newlyAdded.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
if (existing.Count > 0)
|
||||
{
|
||||
cursor[field] = new BsonArray(existing
|
||||
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
return newlyAdded.AsReadOnly();
|
||||
}
|
||||
|
||||
private static Guid AsGuid(BsonValue value)
|
||||
{
|
||||
if (value is null)
|
||||
{
|
||||
return Guid.Empty;
|
||||
}
|
||||
|
||||
return Guid.TryParse(value.ToString(), out var parsed) ? parsed : Guid.Empty;
|
||||
}
|
||||
|
||||
private static void AppendRange(HashSet<Guid> target, IReadOnlyCollection<Guid>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var guid in values)
|
||||
{
|
||||
if (guid != Guid.Empty)
|
||||
{
|
||||
target.Add(guid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendRange(HashSet<string> target, IReadOnlyCollection<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
target.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -7,8 +7,7 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="JsonSchema.Net" Version="5.3.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.5" />
|
||||
<PackageReference Include="MongoDB.Driver.GridFS" Version="2.22.0" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="2.22.0" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<PackageReference Include="AngleSharp" Version="1.1.1" />
|
||||
<PackageReference Include="UglyToad.PdfPig" Version="1.7.0-custom-5" />
|
||||
<PackageReference Include="NuGet.Versioning" Version="6.9.1" />
|
||||
|
||||
@@ -16,4 +16,4 @@
|
||||
|Allow per-request Accept header overrides|BE-Conn-Shared|Source.Common|**DONE** – `SourceFetchRequest.AcceptHeaders` honored by `SourceFetchService` plus unit tests for overrides.|
|
||||
|FEEDCONN-SHARED-HTTP2-001 HTTP version fallback policy|BE-Conn-Shared, Source.Common|Source.Common|**DONE (2025-10-11)** – `AddSourceHttpClient` now honours per-connector HTTP version/ policy, exposes handler customisation, and defaults to downgrade-friendly settings; unit tests cover handler configuration hook.|
|
||||
|FEEDCONN-SHARED-TLS-001 Sovereign trust store support|BE-Conn-Shared, Ops|Source.Common|**DONE (2025-10-11)** – `SourceHttpClientOptions` now exposes `TrustedRootCertificates`, `ServerCertificateCustomValidation`, and `AllowInvalidServerCertificates`, and `AddSourceHttpClient` runs the shared configuration binder so connectors can pull `concelier:httpClients|sources:<name>:http` settings (incl. Offline Kit relative PEM paths via `concelier:offline:root`). Tests cover handler wiring. Ops follow-up: package RU trust roots for Offline Kit distribution.|
|
||||
|FEEDCONN-SHARED-STATE-003 Source state seeding helper|Tools Guild, BE-Conn-MSRC|Tools|**TODO (2025-10-15)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands.|
|
||||
|FEEDCONN-SHARED-STATE-003 Source state seeding helper|Tools Guild, BE-Conn-MSRC|Tools|**DOING (2025-10-19)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands. Prereqs confirmed none (2025-10-19).|
|
||||
|
||||
Reference in New Issue
Block a user