Fix build and code structure improvements. New but essential UI functionality. CI improvements. Documentation improvements. AI module improvements.

2025-12-26 21:54:17 +02:00
parent 335ff7da16
commit c2b9cd8d1f
3717 changed files with 264714 additions and 48202 deletions
--- a/src/Excititor/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs
+++ b/src/Excititor/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs
@@ -243,7 +243,7 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
            LastHeartbeatAt = result.CompletedAt,
            LastHeartbeatStatus = VexWorkerHeartbeatStatus.Succeeded.ToString(),
            LastArtifactHash = result.LastArtifactHash,
-            LastCheckpoint = result.LastCheckpoint,
+            LastCheckpoint = ParseCheckpoint(result.LastCheckpoint),
            FailureCount = 0,
            NextEligibleRun = null,
            LastFailureReason = null
@@ -327,7 +327,7 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient

        await SendRemoteCompletionAsync(
            context,
-            new VexWorkerJobResult(0, 0, state.LastCheckpoint, state.LastArtifactHash, now),
+            new VexWorkerJobResult(0, 0, state.LastCheckpoint?.ToString("O"), state.LastArtifactHash, now),
            cancellationToken,
            success: false,
            failureReason: Truncate($"{errorCode}: {errorMessage}", 256)).ConfigureAwait(false);
@@ -399,7 +399,7 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient

        var updated = state with
        {
-            LastCheckpoint = checkpoint.Cursor,
+            LastCheckpoint = ParseCheckpoint(checkpoint.Cursor),
            LastUpdated = checkpoint.LastProcessedAt ?? now,
            DocumentDigests = checkpoint.ProcessedDigests.IsDefault
                ? ImmutableArray<string>.Empty
@@ -447,7 +447,7 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient

        return new VexWorkerCheckpoint(
            connectorId,
-            state.LastCheckpoint,
+            state.LastCheckpoint?.ToString("O"),
            state.LastUpdated,
            state.DocumentDigests.IsDefault ? ImmutableArray<string>.Empty : state.DocumentDigests,
            state.ResumeTokens.IsEmpty ? ImmutableDictionary<string, string>.Empty : state.ResumeTokens);
@@ -471,6 +471,18 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
            : value[..maxLength];
    }

+    private static DateTimeOffset? ParseCheckpoint(string? checkpoint)
+    {
+        if (string.IsNullOrEmpty(checkpoint))
+        {
+            return null;
+        }
+
+        return DateTimeOffset.TryParse(checkpoint, null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsed)
+            ? parsed
+            : null;
+    }
+
    private int ResolveLeaseSeconds()
    {
        var seconds = (int)Math.Round(_options.Value.DefaultLeaseDuration.TotalSeconds);
--- a/src/Excititor/StellaOps.Excititor.Worker/Program.cs
+++ b/src/Excititor/StellaOps.Excititor.Worker/Program.cs
@@ -6,7 +6,7 @@ using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using StellaOps.Plugin;
-using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
+using StellaOps.Excititor.Connectors.Abstractions;
 using StellaOps.Excititor.Core;
 using StellaOps.Excititor.Core.Aoc;
 using StellaOps.Excititor.Core.Storage;
@@ -14,7 +14,7 @@ using StellaOps.Excititor.Core.Orchestration;
 using StellaOps.Excititor.Formats.CSAF;
 using StellaOps.Excititor.Formats.CycloneDX;
 using StellaOps.Excititor.Formats.OpenVEX;
-using StellaOps.Excititor.Storage.Postgres;
+using StellaOps.Excititor.Persistence.Extensions;
 using StellaOps.Excititor.Worker.Auth;
 using StellaOps.Excititor.Worker.Options;
 using StellaOps.Excititor.Worker.Orchestration;
@@ -43,13 +43,14 @@ services.PostConfigure<VexWorkerOptions>(options =>
        options.Refresh.Enabled = false;
    }
 });
-services.AddRedHatCsafConnector();
+// VEX connectors are loaded via plugin catalog below
+// Direct connector registration removed in favor of plugin-based loading

 services.AddOptions<VexStorageOptions>()
    .Bind(configuration.GetSection("Excititor:Storage"))
    .ValidateOnStart();

-services.AddExcititorPostgresStorage(configuration);
+services.AddExcititorPersistence(configuration);
 services.AddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
 services.TryAddScoped<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
 services.AddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
@@ -91,20 +92,32 @@ services.PostConfigure<VexWorkerOptions>(options =>
        });
    }
 });
+// Load VEX connector plugins
 services.AddSingleton<PluginCatalog>(provider =>
 {
-    var pluginOptions = provider.GetRequiredService<IOptions<VexWorkerPluginOptions>>().Value;
+    var opts = provider.GetRequiredService<IOptions<VexWorkerPluginOptions>>().Value;
    var catalog = new PluginCatalog();

-    var directory = pluginOptions.ResolveDirectory();
+    var directory = opts.ResolveDirectory();
    if (Directory.Exists(directory))
    {
-        catalog.AddFromDirectory(directory, pluginOptions.ResolveSearchPattern());
+        catalog.AddFromDirectory(directory, opts.ResolveSearchPattern());
    }
    else
    {
-        var logger = provider.GetRequiredService<ILogger<Program>>();
-        logger.LogWarning("Excititor worker plugin directory '{Directory}' does not exist; proceeding without external connectors.", directory);
+        // Fallback: try loading from plugins/excititor directory
+        var fallbackPath = Path.Combine(AppContext.BaseDirectory, "plugins", "excititor");
+        if (Directory.Exists(fallbackPath))
+        {
+            catalog.AddFromDirectory(fallbackPath, "StellaOps.Excititor.Connectors.*.dll");
+        }
+        else
+        {
+            var logger = provider.GetRequiredService<ILogger<Program>>();
+            logger.LogWarning(
+                "Excititor worker plugin directory '{Directory}' does not exist; proceeding without external connectors.",
+                directory);
+        }
    }

    return catalog;
@@ -139,4 +152,5 @@ services.AddSingleton<ITenantAuthorityClientFactory, TenantAuthorityClientFactor
 var host = builder.Build();
 await host.RunAsync();

-public partial class Program;
+// Make Program class file-scoped to prevent it from being exposed to referencing assemblies
+file sealed partial class Program;
--- a/src/Excititor/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs
+++ b/src/Excititor/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs
@@ -9,6 +9,7 @@ using StellaOps.Plugin;
 using StellaOps.Excititor.Connectors.Abstractions;
 using StellaOps.Excititor.Core;
 using StellaOps.Excititor.Core.Orchestration;
+using StellaOps.Excititor.Core.Storage;
 using StellaOps.Excititor.Worker.Options;
 using StellaOps.Excititor.Worker.Orchestration;
 using StellaOps.Excititor.Worker.Signature;
@@ -134,7 +135,7 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
        var jobContext = await _orchestratorClient.StartJobAsync(
            _orchestratorOptions.DefaultTenant,
            connector.Id,
-            stateBeforeRun?.LastCheckpoint,
+            stateBeforeRun?.LastCheckpoint?.ToString("O"),
            cancellationToken).ConfigureAwait(false);

        var documentCount = 0;
--- a/src/Excititor/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs
+++ b/src/Excititor/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs
@@ -1,3 +1,5 @@
+#pragma warning disable EXCITITOR001 // Consensus logic is deprecated - refresh service manages VexConsensus during transition
+
 using System.Collections.Concurrent;
 using System.Collections.Immutable;
 using System.Linq;
@@ -9,6 +11,7 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using StellaOps.Excititor.Core;
 using StellaOps.Excititor.Core.Lattice;
+using StellaOps.Excititor.Core.Storage;
 using StellaOps.Excititor.Formats.OpenVEX;
 using StellaOps.Excititor.Policy;
 using StellaOps.Excititor.Worker.Options;
--- a/src/Excititor/StellaOps.Excititor.Worker/Signature/VerifyingVexRawDocumentSink.cs
+++ b/src/Excititor/StellaOps.Excititor.Worker/Signature/VerifyingVexRawDocumentSink.cs
@@ -1,6 +1,7 @@
 using System.Collections.Immutable;
 using System.Globalization;
 using StellaOps.Excititor.Core;
+using StellaOps.Excititor.Core.Storage;

 namespace StellaOps.Excititor.Worker.Signature;

--- a/src/Excititor/StellaOps.Excititor.Worker/Signature/WorkerSignatureVerifier.cs
+++ b/src/Excititor/StellaOps.Excititor.Worker/Signature/WorkerSignatureVerifier.cs
@@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging;
 using StellaOps.Aoc;
 using StellaOps.Excititor.Attestation.Dsse;
 using StellaOps.Excititor.Attestation.Models;
+using StellaOps.Excititor.Core.Dsse;
 using StellaOps.Excititor.Attestation.Verification;
 using StellaOps.Excititor.Core;
 using StellaOps.Excititor.Core.Aoc;
--- a/src/Excititor/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj
+++ b/src/Excititor/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj
@@ -8,16 +8,16 @@
    <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
  </PropertyGroup>
  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" />
  </ItemGroup>
  <ItemGroup>
    <ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
    <ProjectReference Include="../../Concelier/__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
-    <!-- <ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" /> -->
+    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
-    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
+    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Persistence/StellaOps.Excititor.Persistence.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />