feat: Initialize Zastava Webhook service with TLS and Authority authentication

- Added Program.cs to set up the web application with Serilog for logging, health check endpoints, and a placeholder admission endpoint.
- Configured Kestrel server to use TLS 1.3 and handle client certificates appropriately.
- Created StellaOps.Zastava.Webhook.csproj with necessary dependencies including Serilog and Polly.
- Documented tasks in TASKS.md for the Zastava Webhook project, outlining current work and exit criteria for each task.

src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj

@@ -8,11 +8,13 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
+    <ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
+    <ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="9.0.0" />
     <PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
   </ItemGroup>
 </Project>

src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md

@@ -3,5 +3,6 @@ If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md
 | Task | Owner(s) | Depends on | Notes |
 |---|---|---|---|
 |EXCITITOR-CONN-UBUNTU-01-001 – Ubuntu CSAF discovery & channels|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Ubuntu connector project with configurable channel options, catalog loader (network/offline), DI wiring, and discovery unit tests.|
-|EXCITITOR-CONN-UBUNTU-01-002 – Incremental fetch & deduplication|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-UBUNTU-01-001, EXCITITOR-STORAGE-01-003|TODO – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.|
+|EXCITITOR-CONN-UBUNTU-01-002 – Incremental fetch & deduplication|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-UBUNTU-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.|
 |EXCITITOR-CONN-UBUNTU-01-003 – Trust metadata & provenance|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-UBUNTU-01-002, EXCITITOR-POLICY-01-001|TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics.|
+> Remark (2025-10-19, EXCITITOR-CONN-UBUNTU-01-002): Prerequisites EXCITITOR-CONN-UBUNTU-01-001 and EXCITITOR-STORAGE-01-003 verified as **DONE**; advancing to DOING per Wave 0 kickoff.

src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs

@@ -1,16 +1,24 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Globalization;
+using System.Net;
+using System.Net.Http;
 using System.Runtime.CompilerServices;
+using System.Security.Cryptography;
+using System.Text.Json;
 using Microsoft.Extensions.Logging;
 using StellaOps.Excititor.Connectors.Abstractions;
 using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
 using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
 using StellaOps.Excititor.Core;
+using StellaOps.Excititor.Storage.Mongo;
 
 namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF;
 
 public sealed class UbuntuCsafConnector : VexConnectorBase
 {
+    private const string EtagTokenPrefix = "etag:";
+
     private static readonly VexConnectorDescriptor DescriptorInstance = new(
         id: "excititor:ubuntu",
         kind: VexProviderKind.Distro,
@@ -20,6 +28,8 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
     };
 
     private readonly UbuntuCatalogLoader _catalogLoader;
+    private readonly IHttpClientFactory _httpClientFactory;
+    private readonly IVexConnectorStateRepository _stateRepository;
     private readonly IEnumerable<IVexConnectorOptionsValidator<UbuntuConnectorOptions>> _validators;
 
     private UbuntuConnectorOptions? _options;
@@ -27,12 +37,16 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
 
     public UbuntuCsafConnector(
         UbuntuCatalogLoader catalogLoader,
+        IHttpClientFactory httpClientFactory,
+        IVexConnectorStateRepository stateRepository,
         IEnumerable<IVexConnectorOptionsValidator<UbuntuConnectorOptions>> validators,
         ILogger<UbuntuCsafConnector> logger,
         TimeProvider timeProvider)
         : base(DescriptorInstance, logger, timeProvider)
     {
         _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader));
+        _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
+        _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
         _validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<UbuntuConnectorOptions>>();
     }
 
@@ -65,16 +79,424 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
             _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
         }
 
-        LogConnectorEvent(LogLevel.Debug, "fetch", "Ubuntu CSAF discovery ready; channel catalogs handled in subsequent task.", new Dictionary<string, object?>
-        {
-            ["since"] = context.Since?.ToString("O"),
-        });
+        var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
+        var knownTokens = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
+        var digestSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        var tokenSet = new HashSet<string>(StringComparer.Ordinal);
+        var tokenList = new List<string>(knownTokens.Length + 16);
+        var etagMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
 
-        yield break;
+        foreach (var token in knownTokens)
+        {
+            tokenSet.Add(token);
+            tokenList.Add(token);
+            if (TryParseEtagToken(token, out var uri, out var etag))
+            {
+                etagMap[uri] = etag;
+            }
+            else
+            {
+                digestSet.Add(token);
+            }
+        }
+
+        var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue;
+        var latestTimestamp = state?.LastUpdated ?? since;
+        var stateChanged = false;
+
+        foreach (var channel in _catalog.Metadata.Channels)
+        {
+            await foreach (var entry in EnumerateChannelResourcesAsync(channel, cancellationToken).ConfigureAwait(false))
+            {
+                var entryTimestamp = entry.LastModified ?? channel.LastUpdated ?? _catalog.Metadata.GeneratedAt;
+                if (entryTimestamp <= since)
+                {
+                    if (entryTimestamp > latestTimestamp)
+                    {
+                        latestTimestamp = entryTimestamp;
+                    }
+
+                    continue;
+                }
+
+                var expectedDigest = entry.Sha256 is null ? null : NormalizeDigest(entry.Sha256);
+                if (expectedDigest is not null && digestSet.Contains(expectedDigest))
+                {
+                    if (entryTimestamp > latestTimestamp)
+                    {
+                        latestTimestamp = entryTimestamp;
+                    }
+
+                    continue;
+                }
+
+                etagMap.TryGetValue(entry.DocumentUri.ToString(), out var knownEtag);
+
+                var download = await DownloadDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false);
+                if (download is null)
+                {
+                    if (entryTimestamp > latestTimestamp)
+                    {
+                        latestTimestamp = entryTimestamp;
+                    }
+
+                    continue;
+                }
+
+                var document = download.Document;
+                if (!digestSet.Add(document.Digest))
+                {
+                    if (entryTimestamp > latestTimestamp)
+                    {
+                        latestTimestamp = entryTimestamp;
+                    }
+
+                    continue;
+                }
+
+                await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
+                if (tokenSet.Add(document.Digest))
+                {
+                    tokenList.Add(document.Digest);
+                }
+
+                if (!string.IsNullOrWhiteSpace(download.ETag))
+                {
+                    var etagValue = download.ETag!;
+                    etagMap[entry.DocumentUri.ToString()] = etagValue;
+                    var etagToken = CreateEtagToken(entry.DocumentUri, etagValue);
+                    if (tokenSet.Add(etagToken))
+                    {
+                        tokenList.Add(etagToken);
+                    }
+                }
+
+                stateChanged = true;
+                if (entryTimestamp > latestTimestamp)
+                {
+                    latestTimestamp = entryTimestamp;
+                }
+
+                yield return document;
+            }
+        }
+
+        if (stateChanged || latestTimestamp > (state?.LastUpdated ?? DateTimeOffset.MinValue))
+        {
+            var newState = new VexConnectorState(
+                Descriptor.Id,
+                latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp,
+                tokenList.ToImmutableArray());
+
+            await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
+        }
     }
 
     public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
         => throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing.");
 
     public UbuntuCatalogResult? GetCachedCatalog() => _catalog;
+
+    private async IAsyncEnumerable<UbuntuCatalogEntry> EnumerateChannelResourcesAsync(UbuChannelCatalog channel, [EnumeratorCancellation] CancellationToken cancellationToken)
+    {
+        var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName);
+        HttpResponseMessage? response = null;
+        try
+        {
+            response = await client.GetAsync(channel.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
+            response.EnsureSuccessStatusCode();
+
+            await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
+            using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
+
+            if (!document.RootElement.TryGetProperty("resources", out var resourcesElement) || resourcesElement.ValueKind != JsonValueKind.Array)
+            {
+                LogConnectorEvent(LogLevel.Warning, "fetch.catalog.empty", "Ubuntu CSAF channel catalog missing 'resources' array.", new Dictionary<string, object?>
+                {
+                    ["channel"] = channel.Name,
+                    ["catalog"] = channel.CatalogUri.ToString(),
+                });
+                yield break;
+            }
+
+            foreach (var resource in resourcesElement.EnumerateArray())
+            {
+                var type = GetString(resource, "type");
+                if (type is not null && !type.Equals("csaf", StringComparison.OrdinalIgnoreCase))
+                {
+                    continue;
+                }
+
+                var uriText = GetString(resource, "url")
+                    ?? GetString(resource, "canonical")
+                    ?? GetString(resource, "download")
+                    ?? GetString(resource, "uri");
+
+                if (uriText is null || !Uri.TryCreate(uriText, UriKind.Absolute, out var documentUri))
+                {
+                    continue;
+                }
+
+                var sha256 = TryGetHash(resource);
+                var etag = GetString(resource, "etag");
+                var lastModified = ParseDate(resource, "last_modified")
+                    ?? ParseDate(resource, "published")
+                    ?? ParseDate(resource, "released")
+                    ?? channel.LastUpdated;
+                var title = GetString(resource, "title");
+                var version = GetString(resource, "version");
+                var advisoryId = GetString(resource, "id") ?? ExtractAdvisoryId(documentUri, title);
+
+                yield return new UbuntuCatalogEntry(
+                    channel.Name,
+                    advisoryId,
+                    documentUri,
+                    sha256,
+                    etag,
+                    lastModified,
+                    title,
+                    version);
+            }
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            LogConnectorEvent(LogLevel.Warning, "fetch.catalog.failure", "Failed to enumerate Ubuntu CSAF channel catalog.", new Dictionary<string, object?>
+            {
+                ["channel"] = channel.Name,
+                ["catalog"] = channel.CatalogUri.ToString(),
+            }, ex);
+        }
+        finally
+        {
+            response?.Dispose();
+        }
+    }
+
+    private async Task<DownloadResult?> DownloadDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken)
+    {
+        var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName);
+        using var request = new HttpRequestMessage(HttpMethod.Get, entry.DocumentUri);
+        if (!string.IsNullOrWhiteSpace(knownEtag))
+        {
+            request.Headers.IfNoneMatch.TryParseAdd(EnsureQuoted(knownEtag));
+        }
+
+        HttpResponseMessage? response = null;
+        try
+        {
+            response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
+
+            if (response.StatusCode == HttpStatusCode.NotModified)
+            {
+                LogConnectorEvent(LogLevel.Debug, "fetch.document.not_modified", "Ubuntu CSAF document not modified per ETag.", new Dictionary<string, object?>
+                {
+                    ["uri"] = entry.DocumentUri.ToString(),
+                    ["etag"] = knownEtag,
+                });
+                return null;
+            }
+
+            response.EnsureSuccessStatusCode();
+
+            var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
+            if (entry.Sha256 is not null)
+            {
+                var expected = NormalizeDigest(entry.Sha256);
+                var actual = "sha256:" + ComputeSha256Hex(payload);
+                if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase))
+                {
+                    LogConnectorEvent(LogLevel.Warning, "fetch.document.checksum_mismatch", "Ubuntu CSAF document checksum mismatch; skipping document.", new Dictionary<string, object?>
+                    {
+                        ["uri"] = entry.DocumentUri.ToString(),
+                        ["expected"] = expected,
+                        ["actual"] = actual,
+                    });
+                    return null;
+                }
+            }
+
+            var etagHeader = response.Headers.ETag?.Tag;
+            var etagValue = !string.IsNullOrWhiteSpace(etagHeader)
+                ? Unquote(etagHeader!)
+                : entry.ETag is null ? null : Unquote(entry.ETag);
+
+            var metadata = BuildMetadata(builder =>
+            {
+                builder.Add("ubuntu.channel", entry.Channel);
+                builder.Add("ubuntu.uri", entry.DocumentUri.ToString());
+                if (!string.IsNullOrWhiteSpace(entry.AdvisoryId))
+                {
+                    builder.Add("ubuntu.advisoryId", entry.AdvisoryId);
+                }
+
+                if (!string.IsNullOrWhiteSpace(entry.Title))
+                {
+                    builder.Add("ubuntu.title", entry.Title!);
+                }
+
+                if (!string.IsNullOrWhiteSpace(entry.Version))
+                {
+                    builder.Add("ubuntu.version", entry.Version!);
+                }
+
+                if (entry.LastModified is { } modified)
+                {
+                    builder.Add("ubuntu.lastModified", modified.ToString("O"));
+                }
+
+                if (entry.Sha256 is not null)
+                {
+                    builder.Add("ubuntu.sha256", NormalizeDigest(entry.Sha256));
+                }
+
+                if (!string.IsNullOrWhiteSpace(etagValue))
+                {
+                    builder.Add("ubuntu.etag", etagValue!);
+                }
+            });
+
+            var document = CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload, metadata);
+            return new DownloadResult(document, etagValue);
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            LogConnectorEvent(LogLevel.Warning, "fetch.document.failure", "Failed to download Ubuntu CSAF document.", new Dictionary<string, object?>
+            {
+                ["uri"] = entry.DocumentUri.ToString(),
+            }, ex);
+            return null;
+        }
+        finally
+        {
+            response?.Dispose();
+        }
+    }
+
+    private static string NormalizeDigest(string value)
+    {
+        var trimmed = value.Trim();
+        if (trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
+        {
+            trimmed = trimmed[7..];
+        }
+
+        return "sha256:" + trimmed.Replace(" ", string.Empty, StringComparison.Ordinal).ToLowerInvariant();
+    }
+
+    private static string ComputeSha256Hex(ReadOnlySpan<byte> payload)
+    {
+        Span<byte> buffer = stackalloc byte[32];
+        SHA256.HashData(payload, buffer);
+        return Convert.ToHexString(buffer).ToLowerInvariant();
+    }
+
+    private static string? GetString(JsonElement element, string propertyName)
+    {
+        if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
+        {
+            var value = property.GetString();
+            return string.IsNullOrWhiteSpace(value) ? null : value;
+        }
+
+        return null;
+    }
+
+    private static string? TryGetHash(JsonElement resource)
+    {
+        if (resource.TryGetProperty("hashes", out var hashesElement) && hashesElement.ValueKind == JsonValueKind.Object)
+        {
+            if (hashesElement.TryGetProperty("sha256", out var hash) && hash.ValueKind == JsonValueKind.String)
+            {
+                var value = hash.GetString();
+                if (!string.IsNullOrWhiteSpace(value))
+                {
+                    return value;
+                }
+            }
+        }
+
+        return GetString(resource, "sha256");
+    }
+
+    private static DateTimeOffset? ParseDate(JsonElement element, string propertyName)
+    {
+        var text = GetString(element, propertyName);
+        if (text is null)
+        {
+            return null;
+        }
+
+        return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var value)
+            ? value
+            : (DateTimeOffset?)null;
+    }
+
+    private static string ExtractAdvisoryId(Uri uri, string? title)
+    {
+        if (!string.IsNullOrWhiteSpace(title))
+        {
+            return title!;
+        }
+
+        var segments = uri.Segments;
+        if (segments.Length > 0)
+        {
+            var candidate = segments[^1].Trim('/');
+            if (candidate.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
+            {
+                candidate = candidate[..^5];
+            }
+
+            if (!string.IsNullOrWhiteSpace(candidate))
+            {
+                return candidate;
+            }
+        }
+
+        return uri.AbsolutePath.Trim('/');
+    }
+
+    private static string EnsureQuoted(string value)
+    {
+        var trimmed = value.Trim();
+        return trimmed.StartsWith('"') ? trimmed : $"\"{trimmed}\"";
+    }
+
+    private static string Unquote(string value)
+        => value.Trim().Trim('"');
+
+    private static string CreateEtagToken(Uri uri, string etag)
+        => $"{EtagTokenPrefix}{uri}|{etag}";
+
+    private static bool TryParseEtagToken(string token, out string uri, out string etag)
+    {
+        uri = string.Empty;
+        etag = string.Empty;
+        if (!token.StartsWith(EtagTokenPrefix, StringComparison.Ordinal))
+        {
+            return false;
+        }
+
+        var separatorIndex = token.IndexOf('|', EtagTokenPrefix.Length);
+        if (separatorIndex < 0 || separatorIndex == EtagTokenPrefix.Length)
+        {
+            return false;
+        }
+
+        uri = token[EtagTokenPrefix.Length..separatorIndex];
+        etag = token[(separatorIndex + 1)..];
+        return !string.IsNullOrWhiteSpace(uri) && !string.IsNullOrWhiteSpace(etag);
+    }
+
+    private sealed record UbuntuCatalogEntry(
+        string Channel,
+        string? AdvisoryId,
+        Uri DocumentUri,
+        string? Sha256,
+        string? ETag,
+        DateTimeOffset? LastModified,
+        string? Title,
+        string? Version);
+
+    private sealed record DownloadResult(VexRawDocument Document, string? ETag);
 }