Rewrite architecture docs and add Vexer connector template

2025-10-17 19:34:43 +03:00
parent 7bf40b8589
commit e2672ba968
25 changed files with 4885 additions and 777 deletions
--- a/docs/dev/30_VEXER_CONNECTOR_GUIDE.md
+++ b/docs/dev/30_VEXER_CONNECTOR_GUIDE.md
@@ -0,0 +1,220 @@
+# Vexer Connector Packaging Guide
+
+> **Audience:** teams implementing new Vexer provider plug‑ins (CSAF feeds,
+> OpenVEX attestations, etc.)  
+> **Prerequisites:** read `docs/ARCHITECTURE_VEXER.md` and the module
+> `AGENTS.md` in `src/StellaOps.Vexer.Connectors.Abstractions/`.
+
+The Vexer connector SDK gives you:
+
+- `VexConnectorBase` – deterministic logging, SHA‑256 helpers, time provider.
+- `VexConnectorOptionsBinder` – strongly typed YAML/JSON configuration binding.
+- `IVexConnectorOptionsValidator<T>` – custom validation hooks (offline defaults, auth invariants).
+- `VexConnectorDescriptor` & metadata helpers for consistent telemetry.
+
+This guide explains how to package a connector so the Vexer Worker/WebService
+can load it via the plugin host.
+
+---
+
+## 1. Project layout
+
+Start from the template under
+`docs/dev/templates/vexer-connector/`. It contains:
+
+```
+Vexer.MyConnector/
+├── src/
+│   ├── Vexer.MyConnector.csproj
+│   ├── MyConnectorOptions.cs
+│   ├── MyConnector.cs
+│   └── MyConnectorPlugin.cs
+└── manifest/
+    └── connector.manifest.yaml
+```
+
+Key points:
+
+- Target `net10.0`, enable `TreatWarningsAsErrors`, reference the
+  `StellaOps.Vexer.Connectors.Abstractions` project (or NuGet once published).
+- Keep project ID prefix `StellaOps.Vexer.Connectors.<Provider>` so the
+  plugin loader can discover it with the default search pattern.
+
+### 1.1 csproj snippet
+
+```xml
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\StellaOps.Vexer.Connectors.Abstractions\StellaOps.Vexer.Connectors.Abstractions.csproj" />
+  </ItemGroup>
+</Project>
+```
+
+Adjust the `ProjectReference` for your checkout (or switch to a NuGet package
+once published).
+
+---
+
+## 2. Implement the connector
+
+1. **Options model** – create an options POCO with data-annotation attributes.
+   Bind it via `VexConnectorOptionsBinder.Bind<TOptions>` in your connector
+   constructor or `ValidateAsync`.
+2. **Validator** – implement `IVexConnectorOptionsValidator<TOptions>` to add
+   complex checks (e.g., ensure both `clientId` and `clientSecret` are present).
+3. **Connector** – inherit from `VexConnectorBase`. Implement:
+   - `ValidateAsync` – run binder/validators, log configuration summary.
+   - `FetchAsync` – stream raw documents to `context.RawSink`.
+   - `NormalizeAsync` – convert raw documents into `VexClaimBatch` via
+     format-specific normalizers (`context.Normalizers`).
+4. **Plugin adapter** – expose the connector via a plugin entry point so the
+   host can instantiate it.
+
+### 2.1 Options binding example
+
+```csharp
+public sealed class MyConnectorOptions
+{
+    [Required]
+    [Url]
+    public string CatalogUri { get; set; } = default!;
+
+    [Required]
+    public string ApiKey { get; set; } = default!;
+
+    [Range(1, 64)]
+    public int MaxParallelRequests { get; set; } = 4;
+}
+
+public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator<MyConnectorOptions>
+{
+    public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList<string> errors)
+    {
+        if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+        {
+            errors.Add("CatalogUri must use HTTPS.");
+        }
+    }
+}
+```
+
+Bind inside the connector:
+
+```csharp
+private readonly MyConnectorOptions _options;
+
+public MyConnector(VexConnectorDescriptor descriptor, ILogger<MyConnector> logger, TimeProvider timeProvider)
+    : base(descriptor, logger, timeProvider)
+{
+    // `settings` comes from the orchestrator; validators registered via DI.
+    _options = VexConnectorOptionsBinder.Bind<MyConnectorOptions>(
+        descriptor,
+        VexConnectorSettings.Empty,
+        validators: new[] { new MyConnectorOptionsValidator() });
+}
+```
+
+Replace `VexConnectorSettings.Empty` with the actual settings from context
+inside `ValidateAsync`.
+
+---
+
+## 3. Plugin adapter & manifest
+
+Create a simple plugin class that implements
+`StellaOps.Plugin.IConnectorPlugin`. The Worker/WebService plugin host uses
+this contract today.
+
+```csharp
+public sealed class MyConnectorPlugin : IConnectorPlugin
+{
+    private static readonly VexConnectorDescriptor Descriptor =
+        new("vexer:my-provider", VexProviderKind.Vendor, "My Provider VEX");
+
+    public string Name => Descriptor.DisplayName;
+
+    public bool IsAvailable(IServiceProvider services) => true; // inject feature flags if needed
+
+    public IFeedConnector Create(IServiceProvider services)
+    {
+        var logger = services.GetRequiredService<ILogger<MyConnector>>();
+        var timeProvider = services.GetRequiredService<TimeProvider>();
+        return new MyConnector(Descriptor, logger, timeProvider);
+    }
+}
+```
+
+> **Note:** the Vexer Worker currently instantiates connectors through the
+> shared `IConnectorPlugin` contract. Once a dedicated Vexer plugin interface
+> lands you simply swap the base interface; the descriptor/connector code
+> remains unchanged.
+
+Provide a manifest describing the assembly for operational tooling:
+
+```yaml
+# manifest/connector.manifest.yaml
+id: vexer-my-provider
+assembly: StellaOps.Vexer.Connectors.MyProvider.dll
+entryPoint: StellaOps.Vexer.Connectors.MyProvider.MyConnectorPlugin
+description: >
+  Official VEX feed for ExampleCorp products (CSAF JSON, daily updates).
+tags:
+  - vexer
+  - csaf
+  - vendor
+```
+
+Store manifests under `/opt/stella/vexer/plugins/<connector>/manifest/` in
+production so the deployment tooling can inventory and verify plug‑ins.
+
+---
+
+## 4. Packaging workflow
+
+1. `dotnet publish -c Release` → copy the published DLLs to
+   `/opt/stella/vexer/plugins/<Provider>/`.
+2. Place `connector.manifest.yaml` next to the binaries.
+3. Restart the Vexer Worker or WebService (hot reload not supported yet).
+4. Verify logs: `VEX-ConnectorLoader` should list the connector descriptor.
+
+### 4.1 Offline kits
+
+- Add the connector folder (binaries + manifest) to the Offline Kit bundle.
+- Include a `settings.sample.yaml` demonstrating offline-friendly defaults.
+- Document any external dependencies (e.g., SHA mirrors) in the manifest `notes`
+  field.
+
+---
+
+## 5. Testing checklist
+
+- Unit tests around options binding & validators.
+- Integration tests (future `StellaOps.Vexer.Connectors.Abstractions.Tests`)
+  verifying deterministic logging scopes:
+  `logger.BeginScope` should produce `vex.connector.id`, `vex.connector.kind`,
+  and `vex.connector.operation`.
+- Deterministic SHA tests: repeated `CreateRawDocument` calls with identical
+  content must return the same digest.
+
+---
+
+## 6. Reference template
+
+See `docs/dev/templates/vexer-connector/` for the full quick‑start including:
+
+- Sample options class + validator.
+- Connector implementation inheriting from `VexConnectorBase`.
+- Plugin adapter + manifest.
+
+Copy the directory, rename namespaces/IDs, then iterate on provider-specific
+logic.
+
+---
+
+*Last updated: 2025-10-17*
--- a/docs/dev/templates/vexer-connector/manifest/connector.manifest.yaml
+++ b/docs/dev/templates/vexer-connector/manifest/connector.manifest.yaml
@@ -0,0 +1,8 @@
+id: vexer-my-provider
+assembly: StellaOps.Vexer.Connectors.MyProvider.dll
+entryPoint: StellaOps.Vexer.Connectors.MyProvider.MyConnectorPlugin
+description: |
+  Example connector template. Replace metadata before shipping.
+tags:
+  - vexer
+  - template
--- a/docs/dev/templates/vexer-connector/src/MyConnector.cs
+++ b/docs/dev/templates/vexer-connector/src/MyConnector.cs
@@ -0,0 +1,72 @@
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
+using Microsoft.Extensions.Logging;
+using StellaOps.Vexer.Connectors.Abstractions;
+using StellaOps.Vexer.Core;
+
+namespace StellaOps.Vexer.Connectors.MyProvider;
+
+public sealed class MyConnector : VexConnectorBase
+{
+    private readonly IEnumerable<IVexConnectorOptionsValidator<MyConnectorOptions>> _validators;
+    private MyConnectorOptions? _options;
+
+    public MyConnector(VexConnectorDescriptor descriptor, ILogger<MyConnector> logger, TimeProvider timeProvider, IEnumerable<IVexConnectorOptionsValidator<MyConnectorOptions>> validators)
+        : base(descriptor, logger, timeProvider)
+    {
+        _validators = validators;
+    }
+
+    public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
+    {
+        _options = VexConnectorOptionsBinder.Bind(
+            Descriptor,
+            settings,
+            validators: _validators);
+
+        LogConnectorEvent(LogLevel.Information, "validate", "MyConnector configuration loaded.",
+            new Dictionary<string, object?>
+            {
+                ["catalogUri"] = _options.CatalogUri,
+                ["maxParallelRequests"] = _options.MaxParallelRequests,
+            });
+
+        return ValueTask.CompletedTask;
+    }
+
+    public override IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken)
+    {
+        if (_options is null)
+        {
+            throw new InvalidOperationException("Connector not validated.");
+        }
+
+        return FetchInternalAsync(context, cancellationToken);
+    }
+
+    private async IAsyncEnumerable<VexRawDocument> FetchInternalAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
+    {
+        LogConnectorEvent(LogLevel.Information, "fetch", "Fetching catalog window...");
+
+        // Replace with real HTTP logic.
+        await Task.Delay(10, cancellationToken);
+
+        var metadata = BuildMetadata(builder => builder
+            .Add("sourceUri", _options!.CatalogUri)
+            .Add("window", context.Since?.ToString("O") ?? "full"));
+
+        yield return CreateRawDocument(
+            VexDocumentFormat.CsafJson,
+            new Uri($"{_options.CatalogUri.TrimEnd('/')}/sample.json"),
+            new byte[] { 0x7B, 0x7D },
+            metadata);
+    }
+
+    public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
+    {
+        var claims = ImmutableArray<VexClaim>.Empty;
+        var diagnostics = ImmutableDictionary<string, string>.Empty;
+        return ValueTask.FromResult(new VexClaimBatch(document, claims, diagnostics));
+    }
+}
--- a/docs/dev/templates/vexer-connector/src/MyConnectorOptions.cs
+++ b/docs/dev/templates/vexer-connector/src/MyConnectorOptions.cs
@@ -0,0 +1,16 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace StellaOps.Vexer.Connectors.MyProvider;
+
+public sealed class MyConnectorOptions
+{
+    [Required]
+    [Url]
+    public string CatalogUri { get; set; } = default!;
+
+    [Required]
+    public string ApiKey { get; set; } = default!;
+
+    [Range(1, 32)]
+    public int MaxParallelRequests { get; set; } = 4;
+}
--- a/docs/dev/templates/vexer-connector/src/MyConnectorOptionsValidator.cs
+++ b/docs/dev/templates/vexer-connector/src/MyConnectorOptionsValidator.cs
@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+using StellaOps.Vexer.Connectors.Abstractions;
+
+namespace StellaOps.Vexer.Connectors.MyProvider;
+
+public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator<MyConnectorOptions>
+{
+    public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList<string> errors)
+    {
+        if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+        {
+            errors.Add("CatalogUri must use HTTPS.");
+        }
+    }
+}
--- a/docs/dev/templates/vexer-connector/src/MyConnectorPlugin.cs
+++ b/docs/dev/templates/vexer-connector/src/MyConnectorPlugin.cs
@@ -0,0 +1,27 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using StellaOps.Plugin;
+using StellaOps.Vexer.Connectors.Abstractions;
+using StellaOps.Vexer.Core;
+
+namespace StellaOps.Vexer.Connectors.MyProvider;
+
+public sealed class MyConnectorPlugin : IConnectorPlugin
+{
+    private static readonly VexConnectorDescriptor Descriptor = new(
+        id: "vexer:my-provider",
+        kind: VexProviderKind.Vendor,
+        displayName: "My Provider VEX");
+
+    public string Name => Descriptor.DisplayName;
+
+    public bool IsAvailable(IServiceProvider services) => true;
+
+    public IFeedConnector Create(IServiceProvider services)
+    {
+        var logger = services.GetRequiredService<ILogger<MyConnector>>();
+        var timeProvider = services.GetRequiredService<TimeProvider>();
+        var validators = services.GetServices<IVexConnectorOptionsValidator<MyConnectorOptions>>();
+        return new MyConnector(Descriptor, logger, timeProvider, validators);
+    }
+}
--- a/docs/dev/templates/vexer-connector/src/Vexer.MyConnector.csproj
+++ b/docs/dev/templates/vexer-connector/src/Vexer.MyConnector.csproj
@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+  </PropertyGroup>
+  <ItemGroup>
+    <!-- Adjust the relative path when copying this template into a repo -->
+    <ProjectReference Include="..\..\..\..\src\StellaOps.Vexer.Connectors.Abstractions\StellaOps.Vexer.Connectors.Abstractions.csproj" />
+  </ItemGroup>
+</Project>