Resolve Concelier/Excititor merge conflicts
This commit is contained in:
master
220
docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md
Normal file
220
docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md
Normal file
@@ -0,0 +1,220 @@
|
||||
# Excititor Connector Packaging Guide
|
||||
|
||||
> **Audience:** teams implementing new Excititor provider plug‑ins (CSAF feeds,
|
||||
> OpenVEX attestations, etc.)
|
||||
> **Prerequisites:** read `docs/ARCHITECTURE_EXCITITOR.md` and the module
|
||||
> `AGENTS.md` in `src/StellaOps.Excititor.Connectors.Abstractions/`.
|
||||
|
||||
The Excititor connector SDK gives you:
|
||||
|
||||
- `VexConnectorBase` – deterministic logging, SHA‑256 helpers, time provider.
|
||||
- `VexConnectorOptionsBinder` – strongly typed YAML/JSON configuration binding.
|
||||
- `IVexConnectorOptionsValidator<T>` – custom validation hooks (offline defaults, auth invariants).
|
||||
- `VexConnectorDescriptor` & metadata helpers for consistent telemetry.
|
||||
|
||||
This guide explains how to package a connector so the Excititor Worker/WebService
|
||||
can load it via the plugin host.
|
||||
|
||||
---
|
||||
|
||||
## 1. Project layout
|
||||
|
||||
Start from the template under
|
||||
`docs/dev/templates/excititor-connector/`. It contains:
|
||||
|
||||
```
|
||||
Excititor.MyConnector/
|
||||
├── src/
|
||||
│ ├── Excititor.MyConnector.csproj
|
||||
│ ├── MyConnectorOptions.cs
|
||||
│ ├── MyConnector.cs
|
||||
│ └── MyConnectorPlugin.cs
|
||||
└── manifest/
|
||||
└── connector.manifest.yaml
|
||||
```
|
||||
|
||||
Key points:
|
||||
|
||||
- Target `net10.0`, enable `TreatWarningsAsErrors`, reference the
|
||||
`StellaOps.Excititor.Connectors.Abstractions` project (or NuGet once published).
|
||||
- Keep project ID prefix `StellaOps.Excititor.Connectors.<Provider>` so the
|
||||
plugin loader can discover it with the default search pattern.
|
||||
|
||||
### 1.1 csproj snippet
|
||||
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
```
|
||||
|
||||
Adjust the `ProjectReference` for your checkout (or switch to a NuGet package
|
||||
once published).
|
||||
|
||||
---
|
||||
|
||||
## 2. Implement the connector
|
||||
|
||||
1. **Options model** – create an options POCO with data-annotation attributes.
|
||||
Bind it via `VexConnectorOptionsBinder.Bind<TOptions>` in your connector
|
||||
constructor or `ValidateAsync`.
|
||||
2. **Validator** – implement `IVexConnectorOptionsValidator<TOptions>` to add
|
||||
complex checks (e.g., ensure both `clientId` and `clientSecret` are present).
|
||||
3. **Connector** – inherit from `VexConnectorBase`. Implement:
|
||||
- `ValidateAsync` – run binder/validators, log configuration summary.
|
||||
- `FetchAsync` – stream raw documents to `context.RawSink`.
|
||||
- `NormalizeAsync` – convert raw documents into `VexClaimBatch` via
|
||||
format-specific normalizers (`context.Normalizers`).
|
||||
4. **Plugin adapter** – expose the connector via a plugin entry point so the
|
||||
host can instantiate it.
|
||||
|
||||
### 2.1 Options binding example
|
||||
|
||||
```csharp
|
||||
public sealed class MyConnectorOptions
|
||||
{
|
||||
[Required]
|
||||
[Url]
|
||||
public string CatalogUri { get; set; } = default!;
|
||||
|
||||
[Required]
|
||||
public string ApiKey { get; set; } = default!;
|
||||
|
||||
[Range(1, 64)]
|
||||
public int MaxParallelRequests { get; set; } = 4;
|
||||
}
|
||||
|
||||
public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator<MyConnectorOptions>
|
||||
{
|
||||
public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList<string> errors)
|
||||
{
|
||||
if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
errors.Add("CatalogUri must use HTTPS.");
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Bind inside the connector:
|
||||
|
||||
```csharp
|
||||
private readonly MyConnectorOptions _options;
|
||||
|
||||
public MyConnector(VexConnectorDescriptor descriptor, ILogger<MyConnector> logger, TimeProvider timeProvider)
|
||||
: base(descriptor, logger, timeProvider)
|
||||
{
|
||||
// `settings` comes from the orchestrator; validators registered via DI.
|
||||
_options = VexConnectorOptionsBinder.Bind<MyConnectorOptions>(
|
||||
descriptor,
|
||||
VexConnectorSettings.Empty,
|
||||
validators: new[] { new MyConnectorOptionsValidator() });
|
||||
}
|
||||
```
|
||||
|
||||
Replace `VexConnectorSettings.Empty` with the actual settings from context
|
||||
inside `ValidateAsync`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Plugin adapter & manifest
|
||||
|
||||
Create a simple plugin class that implements
|
||||
`StellaOps.Plugin.IConnectorPlugin`. The Worker/WebService plugin host uses
|
||||
this contract today.
|
||||
|
||||
```csharp
|
||||
public sealed class MyConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
private static readonly VexConnectorDescriptor Descriptor =
|
||||
new("excititor:my-provider", VexProviderKind.Vendor, "My Provider VEX");
|
||||
|
||||
public string Name => Descriptor.DisplayName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => true; // inject feature flags if needed
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
var logger = services.GetRequiredService<ILogger<MyConnector>>();
|
||||
var timeProvider = services.GetRequiredService<TimeProvider>();
|
||||
return new MyConnector(Descriptor, logger, timeProvider);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> **Note:** the Excititor Worker currently instantiates connectors through the
|
||||
> shared `IConnectorPlugin` contract. Once a dedicated Excititor plugin interface
|
||||
> lands you simply swap the base interface; the descriptor/connector code
|
||||
> remains unchanged.
|
||||
|
||||
Provide a manifest describing the assembly for operational tooling:
|
||||
|
||||
```yaml
|
||||
# manifest/connector.manifest.yaml
|
||||
id: excititor-my-provider
|
||||
assembly: StellaOps.Excititor.Connectors.MyProvider.dll
|
||||
entryPoint: StellaOps.Excititor.Connectors.MyProvider.MyConnectorPlugin
|
||||
description: >
|
||||
Official VEX feed for ExampleCorp products (CSAF JSON, daily updates).
|
||||
tags:
|
||||
- excititor
|
||||
- csaf
|
||||
- vendor
|
||||
```
|
||||
|
||||
Store manifests under `/opt/stella/excititor/plugins/<connector>/manifest/` in
|
||||
production so the deployment tooling can inventory and verify plug‑ins.
|
||||
|
||||
---
|
||||
|
||||
## 4. Packaging workflow
|
||||
|
||||
1. `dotnet publish -c Release` → copy the published DLLs to
|
||||
`/opt/stella/excititor/plugins/<Provider>/`.
|
||||
2. Place `connector.manifest.yaml` next to the binaries.
|
||||
3. Restart the Excititor Worker or WebService (hot reload not supported yet).
|
||||
4. Verify logs: `VEX-ConnectorLoader` should list the connector descriptor.
|
||||
|
||||
### 4.1 Offline kits
|
||||
|
||||
- Add the connector folder (binaries + manifest) to the Offline Kit bundle.
|
||||
- Include a `settings.sample.yaml` demonstrating offline-friendly defaults.
|
||||
- Document any external dependencies (e.g., SHA mirrors) in the manifest `notes`
|
||||
field.
|
||||
|
||||
---
|
||||
|
||||
## 5. Testing checklist
|
||||
|
||||
- Unit tests around options binding & validators.
|
||||
- Integration tests (future `StellaOps.Excititor.Connectors.Abstractions.Tests`)
|
||||
verifying deterministic logging scopes:
|
||||
`logger.BeginScope` should produce `vex.connector.id`, `vex.connector.kind`,
|
||||
and `vex.connector.operation`.
|
||||
- Deterministic SHA tests: repeated `CreateRawDocument` calls with identical
|
||||
content must return the same digest.
|
||||
|
||||
---
|
||||
|
||||
## 6. Reference template
|
||||
|
||||
See `docs/dev/templates/excititor-connector/` for the full quick‑start including:
|
||||
|
||||
- Sample options class + validator.
|
||||
- Connector implementation inheriting from `VexConnectorBase`.
|
||||
- Plugin adapter + manifest.
|
||||
|
||||
Copy the directory, rename namespaces/IDs, then iterate on provider-specific
|
||||
logic.
|
||||
|
||||
---
|
||||
|
||||
*Last updated: 2025-10-17*
|
||||
@@ -1,210 +1,212 @@
|
||||
# Authority Plug-in Developer Guide
|
||||
|
||||
> **Status:** Updated 2025-10-11 (AUTHPLUG-DOCS-01-001) with lifecycle + limiter diagrams and refreshed rate-limit guidance aligned to PLG6 acceptance criteria.
|
||||
|
||||
## 1. Overview
|
||||
Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Feedser plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows:
|
||||
|
||||
- **Use cases:** integrate corporate directories (LDAP/AD), delegate to external IDPs, enforce bespoke password/lockout policies, or add client provisioning automation.
|
||||
- **Constraints:** plug-ins load only during service start (no hot-reload), must function without outbound internet access, and must emit deterministic results for identical configuration and input data.
|
||||
- **Ship targets:** target the same .NET 10 preview as the host, honour offline-first requirements, and provide clear diagnostics so operators can triage issues from `/ready`.
|
||||
|
||||
## 2. Architecture Snapshot
|
||||
Authority hosts follow a deterministic plug-in lifecycle. The flow below can be rendered as a sequence diagram in the final authored documentation, but all touchpoints are described here for offline viewers:
|
||||
|
||||
1. **Configuration load** – `AuthorityPluginConfigurationLoader` resolves YAML manifests under `etc/authority.plugins/`.
|
||||
2. **Assembly discovery** – the shared `PluginHost` scans `PluginBinaries/Authority` for `StellaOps.Authority.Plugin.*.dll` assemblies.
|
||||
3. **Registrar execution** – each assembly is searched for `IAuthorityPluginRegistrar` implementations. Registrars bind options, register services, and optionally queue bootstrap tasks.
|
||||
4. **Runtime** – the host resolves `IIdentityProviderPlugin` instances, uses capability metadata to decide which OAuth grants to expose, and invokes health checks for readiness endpoints.
|
||||
|
||||
|
||||
|
||||
_Source:_ `docs/assets/authority/authority-plugin-lifecycle.mmd`
|
||||
|
||||
**Data persistence primer:** the standard Mongo-backed plugin stores users in collections named `authority_users_<pluginName>` and lockout metadata in embedded documents. Additional plugins must document their storage layout and provide deterministic collection naming to honour the Offline Kit replication process.
|
||||
|
||||
## 3. Capability Metadata
|
||||
Capability flags let the host reason about what your plug-in supports:
|
||||
|
||||
- Declare capabilities in your descriptor using the string constants from `AuthorityPluginCapabilities` (`password`, `mfa`, `clientProvisioning`, `bootstrap`). The configuration loader now validates these tokens and rejects unknown values at startup.
|
||||
- `AuthorityIdentityProviderCapabilities.FromCapabilities` projects those strings into strongly typed booleans (`SupportsPassword`, etc.). Authority Core will use these flags when wiring flows such as the password grant. Built-in plugins (e.g., Standard) will fail fast or force-enable required capabilities if the descriptor is misconfigured, so keep manifests accurate.
|
||||
- Typical configuration (`etc/authority.plugins/standard.yaml`):
|
||||
```yaml
|
||||
plugins:
|
||||
descriptors:
|
||||
standard:
|
||||
assemblyName: "StellaOps.Authority.Plugin.Standard"
|
||||
capabilities:
|
||||
- password
|
||||
- bootstrap
|
||||
```
|
||||
- Only declare a capability if the plug-in genuinely implements it. For example, if `SupportsClientProvisioning` is `true`, the plug-in must supply a working `IClientProvisioningStore`.
|
||||
|
||||
**Operational reminder:** the Authority host surfaces capability summaries during startup (see `AuthorityIdentityProviderRegistry` log lines). Use those logs during smoke tests to ensure manifests align with expectations.
|
||||
|
||||
**Configuration path normalisation:** Manifest-relative paths (e.g., `tokenSigning.keyDirectory: "../keys"`) are resolved against the YAML file location and environment variables are expanded before validation. Plug-ins should expect to receive an absolute, canonical path when options are injected.
|
||||
|
||||
**Password policy guardrails:** The Standard registrar logs a warning when a plug-in weakens the default password policy (minimum length or required character classes). Keep overrides at least as strong as the compiled defaults—operators treat the warning as an actionable security deviation.
|
||||
|
||||
## 4. Project Scaffold
|
||||
- Target **.NET 10 preview**, enable nullable, treat warnings as errors, and mark Authority plug-ins with `<IsAuthorityPlugin>true</IsAuthorityPlugin>`.
|
||||
- Minimum references:
|
||||
- `StellaOps.Authority.Plugins.Abstractions` (contracts & capability helpers)
|
||||
- `StellaOps.Plugin` (hosting/DI helpers)
|
||||
- `StellaOps.Auth.*` libraries as needed for shared token utilities (optional today).
|
||||
- Example `.csproj` (trimmed from `StellaOps.Authority.Plugin.Standard`):
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<IsAuthorityPlugin>true</IsAuthorityPlugin>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Authority.Plugins.Abstractions\StellaOps.Authority.Plugins.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\..\StellaOps.Plugin\StellaOps.Plugin.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
```
|
||||
(Add other references—e.g., MongoDB driver, shared auth libraries—according to your implementation.)
|
||||
|
||||
## 5. Implementing `IAuthorityPluginRegistrar`
|
||||
- Create a parameterless registrar class that returns your plug-in type name via `PluginType`.
|
||||
- Use `AuthorityPluginRegistrationContext` to:
|
||||
- Bind options (`AddOptions<T>(pluginName).Bind(...)`).
|
||||
- Register singletons for stores/enrichers using manifest metadata.
|
||||
- Register any hosted bootstrap tasks (e.g., seed admin users).
|
||||
- Always validate configuration inside `PostConfigure` and throw meaningful `InvalidOperationException` to fail fast during startup.
|
||||
- Use the provided `ILoggerFactory` from DI; avoid static loggers or console writes.
|
||||
- Example skeleton:
|
||||
```csharp
|
||||
internal sealed class MyPluginRegistrar : IAuthorityPluginRegistrar
|
||||
{
|
||||
public string PluginType => "my-custom";
|
||||
|
||||
public void Register(AuthorityPluginRegistrationContext context)
|
||||
{
|
||||
var name = context.Plugin.Manifest.Name;
|
||||
|
||||
context.Services.AddOptions<MyPluginOptions>(name)
|
||||
.Bind(context.Plugin.Configuration)
|
||||
.PostConfigure(opts => opts.Validate(name));
|
||||
|
||||
context.Services.AddSingleton<IIdentityProviderPlugin>(sp =>
|
||||
new MyIdentityProvider(context.Plugin, sp.GetRequiredService<MyCredentialStore>(),
|
||||
sp.GetRequiredService<MyClaimsEnricher>(),
|
||||
sp.GetRequiredService<ILogger<MyIdentityProvider>>()));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 6. Identity Provider Surface
|
||||
- Implement `IIdentityProviderPlugin` to expose:
|
||||
- `IUserCredentialStore` for password validation and user CRUD.
|
||||
- `IClaimsEnricher` to append roles/attributes onto issued principals.
|
||||
- Optional `IClientProvisioningStore` for machine-to-machine clients.
|
||||
- `AuthorityIdentityProviderCapabilities` to advertise supported flows.
|
||||
- Password guidance:
|
||||
- Standard plug-in hashes via `ICryptoProvider` using Argon2id by default and emits PHC-compliant strings. Successful PBKDF2 logins trigger automatic rehashes so migrations complete gradually. See `docs/security/password-hashing.md` for tuning advice.
|
||||
- Enforce password policies before hashing to avoid storing weak credentials.
|
||||
- Health checks should probe backing stores (e.g., Mongo `ping`) and return `AuthorityPluginHealthResult` so `/ready` can surface issues.
|
||||
- When supporting additional factors (e.g., TOTP), implement `SupportsMfa` and document the enrolment flow for resource servers.
|
||||
|
||||
## 7. Configuration & Secrets
|
||||
- Authority looks for manifests under `etc/authority.plugins/`. Each YAML file maps directly to a plug-in name.
|
||||
- Support environment overrides using `STELLAOPS_AUTHORITY_PLUGINS__DESCRIPTORS__<NAME>__...`.
|
||||
- Never store raw secrets in git: allow operators to supply them via `.local.yaml`, environment variables, or injected secret files. Document which keys are mandatory.
|
||||
- Validate configuration as soon as the registrar runs; use explicit error messages to guide operators. The Standard plug-in now enforces complete bootstrap credentials (username + password) and positive lockout windows via `StandardPluginOptions.Validate`.
|
||||
- Cross-reference bootstrap workflows with `docs/ops/authority_bootstrap.md` (to be published alongside CORE6) so operators can reuse the same payload formats for manual provisioning.
|
||||
- `passwordHashing` inherits defaults from `authority.security.passwordHashing`. Override only when hardware constraints differ per plug-in:
|
||||
```yaml
|
||||
passwordHashing:
|
||||
algorithm: Argon2id
|
||||
memorySizeInKib: 19456
|
||||
iterations: 2
|
||||
parallelism: 1
|
||||
```
|
||||
Invalid values (≤0) fail fast during startup, and legacy PBKDF2 hashes rehash automatically once the new algorithm succeeds.
|
||||
|
||||
### 7.1 Token Persistence Contract
|
||||
- The host automatically persists every issued principal (access, refresh, device, authorization code) in `authority_tokens`. Plug-in code **must not** bypass this store; use the provided `IAuthorityTokenStore` helpers when implementing custom flows.
|
||||
- When a plug-in disables a subject or client outside the standard handlers, call `IAuthorityTokenStore.UpdateStatusAsync(...)` for each affected token so revocation bundles stay consistent.
|
||||
- Supply machine-friendly `revokedReason` codes (`compromised`, `rotation`, `policy`, `lifecycle`, etc.) and optional `revokedMetadata` entries when invalidating credentials. These flow straight into `revocation-bundle.json` and should remain deterministic.
|
||||
- Token scopes should be normalised (trimmed, unique, ordinal sort) before returning from plug-in verification paths. `TokenPersistenceHandlers` will keep that ordering for downstream consumers.
|
||||
|
||||
### 7.2 Claims & Enrichment Checklist
|
||||
- Authority always sets the OpenID Connect basics: `sub`, `client_id`, `preferred_username`, optional `name`, and `role` (for password flows). Plug-ins must use `IClaimsEnricher` to append additional claims in a **deterministic** order (sort arrays, normalise casing) so resource servers can rely on stable shapes.
|
||||
- Recommended enrichment keys:
|
||||
- `stellaops.realm` – plug-in/tenant identifier so services can scope policies.
|
||||
- `stellaops.subject.type` – values such as `human`, `service`, `bootstrap`.
|
||||
- `groups` / `projects` – sorted arrays describing operator entitlements.
|
||||
- Claims visible in tokens should mirror what `/token` and `/userinfo` emit. Avoid injecting sensitive PII directly; mark values with `ClassifiedString.Personal` inside the plug-in so audit sinks can tag them appropriately.
|
||||
- For client-credential flows, remember to enrich both the client principal and the validation path (`TokenValidationHandlers`) so refresh flows keep the same metadata.
|
||||
|
||||
### 7.3 Revocation Bundles & Reasons
|
||||
- Use `IAuthorityRevocationStore` to record subject/client/token revocations when credentials are deleted or rotated. Stick to the standard categories (`token`, `subject`, `client`, `key`).
|
||||
- Include a deterministic `reason` string and optional `reasonDescription` so operators understand *why* a subject was revoked when inspecting bundles offline.
|
||||
- Plug-ins should populate `metadata` with stable keys (e.g., `revokedBy`, `sourcePlugin`, `ticketId`) to simplify SOC correlation. The keys must be lowercase, ASCII, and free of secrets—bundles are mirrored to air-gapped agents.
|
||||
|
||||
## 8. Rate Limiting & Lockout Interplay
|
||||
Rate limiting and account lockouts are complementary controls. Plug-ins must surface both deterministically so operators can correlate limiter hits with credential rejections.
|
||||
|
||||
**Baseline quotas** (from `docs/dev/authority-rate-limit-tuning-outline.md`):
|
||||
|
||||
| Endpoint | Default policy | Notes |
|
||||
|----------|----------------|-------|
|
||||
| `/token` | 30 requests / 60s, queue 0 | Drop to 10/60s for untrusted ranges; raise only with WAF + monitoring. |
|
||||
| `/authorize` | 60 requests / 60s, queue 10 | Reduce carefully; interactive UX depends on headroom. |
|
||||
| `/internal/*` | Disabled by default; recommended 5/60s when enabled | Keep queue 0 for bootstrap APIs. |
|
||||
|
||||
**Retry metadata:** The middleware stamps `Retry-After` plus tags `authority.client_id`, `authority.remote_ip`, and `authority.endpoint`. Plug-ins should keep these tags intact when crafting responses or telemetry so dashboards remain consistent.
|
||||
|
||||
**Lockout counters:** Treat lockouts as **subject-scoped** decisions. When multiple instances update counters, reuse the deterministic tie-breakers documented in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` (freshness overrides, precedence, and stable hashes) to avoid divergent lockout states across replicas.
|
||||
|
||||
**Alerting hooks:** Emit structured logs/metrics when either the limiter or credential store rejects access. Suggested gauges include `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` and any custom `auth.plugins.<pluginName>.lockouts_total` counter.
|
||||
|
||||
|
||||
|
||||
_Source:_ `docs/assets/authority/authority-rate-limit-flow.mmd`
|
||||
|
||||
## 9. Logging, Metrics, and Diagnostics
|
||||
- Always log via the injected `ILogger<T>`; include `pluginName` and correlation IDs where available.
|
||||
- Activity/metric names should align with `AuthorityTelemetry` constants (`service.name=stellaops-authority`).
|
||||
- Expose additional diagnostics via structured logging rather than writing custom HTTP endpoints; the host will integrate these into `/health` and `/ready`.
|
||||
- Emit metrics with stable names (`auth.plugins.<pluginName>.*`) when introducing custom instrumentation; coordinate with the Observability guild to reserve prefixes.
|
||||
|
||||
## 10. Testing & Tooling
|
||||
- Unit tests: use Mongo2Go (or similar) to exercise credential stores without hitting production infrastructure (`StandardUserCredentialStoreTests` is a template).
|
||||
- Determinism: fix timestamps to UTC and sort outputs consistently; avoid random GUIDs unless stable.
|
||||
- Smoke tests: launch `dotnet run --project src/StellaOps.Authority/StellaOps.Authority` with your plug-in under `PluginBinaries/Authority` and verify `/ready`.
|
||||
- Example verification snippet:
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task VerifyPasswordAsync_ReturnsSuccess()
|
||||
{
|
||||
var store = CreateCredentialStore();
|
||||
await store.UpsertUserAsync(new AuthorityUserRegistration("alice", "Pa55!", null, null, false,
|
||||
Array.Empty<string>(), new Dictionary<string, string?>()), CancellationToken.None);
|
||||
|
||||
var result = await store.VerifyPasswordAsync("alice", "Pa55!", CancellationToken.None);
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.True(result.User?.Roles.Count == 0);
|
||||
}
|
||||
```
|
||||
|
||||
## 11. Packaging & Delivery
|
||||
- Output assembly should follow `StellaOps.Authority.Plugin.<Name>.dll` so the host’s search pattern picks it up.
|
||||
- Place the compiled DLL plus dependencies under `PluginBinaries/Authority` for offline deployments; include hashes/signatures in release notes (Security Guild guidance forthcoming).
|
||||
- Document any external prerequisites (e.g., CA cert bundle) in your plug-in README.
|
||||
- Update `etc/authority.plugins/<plugin>.yaml` samples and include deterministic SHA256 hashes for optional bootstrap payloads when distributing Offline Kit artefacts.
|
||||
|
||||
## 12. Checklist & Handoff
|
||||
- ✅ Capabilities declared and validated in automated tests.
|
||||
- ✅ Bootstrap workflows documented (if `bootstrap` capability used) and repeatable.
|
||||
- ✅ Local smoke test + unit/integration suites green (`dotnet test`).
|
||||
- ✅ Operational docs updated: configuration keys, secrets guidance, troubleshooting.
|
||||
- Submit the developer guide update referencing PLG6/DOC4 and tag DevEx + Docs reviewers for sign-off.
|
||||
|
||||
---
|
||||
Mermaid sources for the embedded diagrams live under `docs/assets/authority/`. Regenerate the SVG assets with your preferred renderer before committing future updates so the visuals stay in sync with the `.mmd` definitions.
|
||||
# Authority Plug-in Developer Guide
|
||||
|
||||
> **Status:** Updated 2025-10-11 (AUTHPLUG-DOCS-01-001) with lifecycle + limiter diagrams and refreshed rate-limit guidance aligned to PLG6 acceptance criteria.
|
||||
|
||||
## 1. Overview
|
||||
Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Concelier plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows:
|
||||
|
||||
- **Use cases:** integrate corporate directories (LDAP/AD)[^ldap-rfc], delegate to external IDPs, enforce bespoke password/lockout policies, or add client provisioning automation.
|
||||
- **Constraints:** plug-ins load only during service start (no hot-reload), must function without outbound internet access, and must emit deterministic results for identical configuration input.
|
||||
- **Ship targets:** build against the host’s .NET 10 preview SDK, honour offline-first requirements, and surface actionable diagnostics so operators can triage issues from `/ready`.
|
||||
|
||||
## 2. Architecture Snapshot
|
||||
Authority hosts follow a deterministic plug-in lifecycle. The exported diagram (`docs/assets/authority/authority-plugin-lifecycle.svg`) mirrors the steps below; regenerate it from the Mermaid source if you update the flow.
|
||||
|
||||
1. **Configuration load** – `AuthorityPluginConfigurationLoader` resolves YAML manifests under `etc/authority.plugins/`.
|
||||
2. **Assembly discovery** – the shared `PluginHost` scans `StellaOps.Authority.PluginBinaries` for `StellaOps.Authority.Plugin.*.dll` assemblies.
|
||||
3. **Registrar execution** – each assembly is searched for `IAuthorityPluginRegistrar` implementations. Registrars bind options, register services, and optionally queue bootstrap tasks.
|
||||
4. **Runtime** – the host resolves `IIdentityProviderPlugin` instances, uses capability metadata to decide which OAuth grants to expose, and invokes health checks for readiness endpoints.
|
||||
|
||||
|
||||
|
||||
_Source:_ `docs/assets/authority/authority-plugin-lifecycle.mmd`
|
||||
|
||||
**Data persistence primer:** the standard Mongo-backed plugin stores users in collections named `authority_users_<pluginName>` and lockout metadata in embedded documents. Additional plugins must document their storage layout and provide deterministic collection naming to honour the Offline Kit replication process.
|
||||
|
||||
## 3. Capability Metadata
|
||||
Capability flags let the host reason about what your plug-in supports:
|
||||
|
||||
- Declare capabilities in your descriptor using the string constants from `AuthorityPluginCapabilities` (`password`, `mfa`, `clientProvisioning`, `bootstrap`). The configuration loader now validates these tokens and rejects unknown values at startup.
|
||||
- `AuthorityIdentityProviderCapabilities.FromCapabilities` projects those strings into strongly typed booleans (`SupportsPassword`, etc.). Authority Core will use these flags when wiring flows such as the password grant. Built-in plugins (e.g., Standard) will fail fast or force-enable required capabilities if the descriptor is misconfigured, so keep manifests accurate.
|
||||
- Typical configuration (`etc/authority.plugins/standard.yaml`):
|
||||
```yaml
|
||||
plugins:
|
||||
descriptors:
|
||||
standard:
|
||||
assemblyName: "StellaOps.Authority.Plugin.Standard"
|
||||
capabilities:
|
||||
- password
|
||||
- bootstrap
|
||||
```
|
||||
- Only declare a capability if the plug-in genuinely implements it. For example, if `SupportsClientProvisioning` is `true`, the plug-in must supply a working `IClientProvisioningStore`.
|
||||
|
||||
**Operational reminder:** the Authority host surfaces capability summaries during startup (see `AuthorityIdentityProviderRegistry` log lines). Use those logs during smoke tests to ensure manifests align with expectations.
|
||||
|
||||
**Configuration path normalisation:** Manifest-relative paths (e.g., `tokenSigning.keyDirectory: "../keys"`) are resolved against the YAML file location and environment variables are expanded before validation. Plug-ins should expect to receive an absolute, canonical path when options are injected.
|
||||
|
||||
**Password policy guardrails:** The Standard registrar logs a warning when a plug-in weakens the default password policy (minimum length or required character classes). Keep overrides at least as strong as the compiled defaults—operators treat the warning as an actionable security deviation.
|
||||
|
||||
## 4. Project Scaffold
|
||||
- Target **.NET 10 preview**, enable nullable, treat warnings as errors, and mark Authority plug-ins with `<IsAuthorityPlugin>true</IsAuthorityPlugin>`.
|
||||
- Minimum references:
|
||||
- `StellaOps.Authority.Plugins.Abstractions` (contracts & capability helpers)
|
||||
- `StellaOps.Plugin` (hosting/DI helpers)
|
||||
- `StellaOps.Auth.*` libraries as needed for shared token utilities (optional today).
|
||||
- Example `.csproj` (trimmed from `StellaOps.Authority.Plugin.Standard`):
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<IsAuthorityPlugin>true</IsAuthorityPlugin>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Authority.Plugins.Abstractions\StellaOps.Authority.Plugins.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\..\StellaOps.Plugin\StellaOps.Plugin.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
```
|
||||
(Add other references—e.g., MongoDB driver, shared auth libraries—according to your implementation.)
|
||||
|
||||
## 5. Implementing `IAuthorityPluginRegistrar`
|
||||
- Create a parameterless registrar class that returns your plug-in type name via `PluginType`.
|
||||
- Use `AuthorityPluginRegistrationContext` to:
|
||||
- Bind options (`AddOptions<T>(pluginName).Bind(...)`).
|
||||
- Register singletons for stores/enrichers using manifest metadata.
|
||||
- Register any hosted bootstrap tasks (e.g., seed admin users).
|
||||
- Always validate configuration inside `PostConfigure` and throw meaningful `InvalidOperationException` to fail fast during startup.
|
||||
- Use the provided `ILoggerFactory` from DI; avoid static loggers or console writes.
|
||||
- Example skeleton:
|
||||
```csharp
|
||||
internal sealed class MyPluginRegistrar : IAuthorityPluginRegistrar
|
||||
{
|
||||
public string PluginType => "my-custom";
|
||||
|
||||
public void Register(AuthorityPluginRegistrationContext context)
|
||||
{
|
||||
var name = context.Plugin.Manifest.Name;
|
||||
|
||||
context.Services.AddOptions<MyPluginOptions>(name)
|
||||
.Bind(context.Plugin.Configuration)
|
||||
.PostConfigure(opts => opts.Validate(name));
|
||||
|
||||
context.Services.AddSingleton<IIdentityProviderPlugin>(sp =>
|
||||
new MyIdentityProvider(context.Plugin, sp.GetRequiredService<MyCredentialStore>(),
|
||||
sp.GetRequiredService<MyClaimsEnricher>(),
|
||||
sp.GetRequiredService<ILogger<MyIdentityProvider>>()));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 6. Identity Provider Surface
|
||||
- Implement `IIdentityProviderPlugin` to expose:
|
||||
- `IUserCredentialStore` for password validation and user CRUD.
|
||||
- `IClaimsEnricher` to append roles/attributes onto issued principals.
|
||||
- Optional `IClientProvisioningStore` for machine-to-machine clients.
|
||||
- `AuthorityIdentityProviderCapabilities` to advertise supported flows.
|
||||
- Password guidance:
|
||||
- Standard plug-in hashes via `ICryptoProvider` using Argon2id by default and emits PHC-compliant strings. Successful PBKDF2 logins trigger automatic rehashes so migrations complete gradually. See `docs/security/password-hashing.md` for tuning advice.
|
||||
- Enforce password policies before hashing to avoid storing weak credentials.
|
||||
- Health checks should probe backing stores (e.g., Mongo `ping`) and return `AuthorityPluginHealthResult` so `/ready` can surface issues.
|
||||
- When supporting additional factors (e.g., TOTP), implement `SupportsMfa` and document the enrolment flow for resource servers.
|
||||
|
||||
## 7. Configuration & Secrets
|
||||
- Authority looks for manifests under `etc/authority.plugins/`. Each YAML file maps directly to a plug-in name.
|
||||
- Support environment overrides using `STELLAOPS_AUTHORITY_PLUGINS__DESCRIPTORS__<NAME>__...`.
|
||||
- Never store raw secrets in git: allow operators to supply them via `.local.yaml`, environment variables, or injected secret files. Document which keys are mandatory.
|
||||
- Validate configuration as soon as the registrar runs; use explicit error messages to guide operators. The Standard plug-in now enforces complete bootstrap credentials (username + password) and positive lockout windows via `StandardPluginOptions.Validate`.
|
||||
- Cross-reference bootstrap workflows with `docs/ops/authority_bootstrap.md` (to be published alongside CORE6) so operators can reuse the same payload formats for manual provisioning.
|
||||
- `passwordHashing` inherits defaults from `authority.security.passwordHashing`. Override only when hardware constraints differ per plug-in:
|
||||
```yaml
|
||||
passwordHashing:
|
||||
algorithm: Argon2id
|
||||
memorySizeInKib: 19456
|
||||
iterations: 2
|
||||
parallelism: 1
|
||||
```
|
||||
Invalid values (≤0) fail fast during startup, and legacy PBKDF2 hashes rehash automatically once the new algorithm succeeds.
|
||||
|
||||
### 7.1 Token Persistence Contract
|
||||
- The host automatically persists every issued principal (access, refresh, device, authorization code) in `authority_tokens`. Plug-in code **must not** bypass this store; use the provided `IAuthorityTokenStore` helpers when implementing custom flows.
|
||||
- When a plug-in disables a subject or client outside the standard handlers, call `IAuthorityTokenStore.UpdateStatusAsync(...)` for each affected token so revocation bundles stay consistent.
|
||||
- Supply machine-friendly `revokedReason` codes (`compromised`, `rotation`, `policy`, `lifecycle`, etc.) and optional `revokedMetadata` entries when invalidating credentials. These flow straight into `revocation-bundle.json` and should remain deterministic.
|
||||
- Token scopes should be normalised (trimmed, unique, ordinal sort) before returning from plug-in verification paths. `TokenPersistenceHandlers` will keep that ordering for downstream consumers.
|
||||
|
||||
### 7.2 Claims & Enrichment Checklist
|
||||
- Authority always sets the OpenID Connect basics: `sub`, `client_id`, `preferred_username`, optional `name`, and `role` (for password flows). Plug-ins must use `IClaimsEnricher` to append additional claims in a **deterministic** order (sort arrays, normalise casing) so resource servers can rely on stable shapes.
|
||||
- Recommended enrichment keys:
|
||||
- `stellaops.realm` – plug-in/tenant identifier so services can scope policies.
|
||||
- `stellaops.subject.type` – values such as `human`, `service`, `bootstrap`.
|
||||
- `groups` / `projects` – sorted arrays describing operator entitlements.
|
||||
- Claims visible in tokens should mirror what `/token` and `/userinfo` emit. Avoid injecting sensitive PII directly; mark values with `ClassifiedString.Personal` inside the plug-in so audit sinks can tag them appropriately.
|
||||
- For client-credential flows, remember to enrich both the client principal and the validation path (`TokenValidationHandlers`) so refresh flows keep the same metadata.
|
||||
|
||||
### 7.3 Revocation Bundles & Reasons
|
||||
- Use `IAuthorityRevocationStore` to record subject/client/token revocations when credentials are deleted or rotated. Stick to the standard categories (`token`, `subject`, `client`, `key`).
|
||||
- Include a deterministic `reason` string and optional `reasonDescription` so operators understand *why* a subject was revoked when inspecting bundles offline.
|
||||
- Plug-ins should populate `metadata` with stable keys (e.g., `revokedBy`, `sourcePlugin`, `ticketId`) to simplify SOC correlation. The keys must be lowercase, ASCII, and free of secrets—bundles are mirrored to air-gapped agents.
|
||||
|
||||
## 8. Rate Limiting & Lockout Interplay
|
||||
Rate limiting and account lockouts are complementary controls. Plug-ins must surface both deterministically so operators can correlate limiter hits with credential rejections.
|
||||
|
||||
**Baseline quotas** (from `docs/dev/authority-rate-limit-tuning-outline.md`):
|
||||
|
||||
| Endpoint | Default policy | Notes |
|
||||
|----------|----------------|-------|
|
||||
| `/token` | 30 requests / 60s, queue 0 | Drop to 10/60s for untrusted ranges; raise only with WAF + monitoring. |
|
||||
| `/authorize` | 60 requests / 60s, queue 10 | Reduce carefully; interactive UX depends on headroom. |
|
||||
| `/internal/*` | Disabled by default; recommended 5/60s when enabled | Keep queue 0 for bootstrap APIs. |
|
||||
|
||||
**Retry metadata:** The middleware stamps `Retry-After` plus tags `authority.client_id`, `authority.remote_ip`, and `authority.endpoint`. Plug-ins should keep these tags intact when crafting responses or telemetry so dashboards remain consistent.
|
||||
|
||||
**Lockout counters:** Treat lockouts as **subject-scoped** decisions. When multiple instances update counters, reuse the deterministic tie-breakers documented in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` (freshness overrides, precedence, and stable hashes) to avoid divergent lockout states across replicas.
|
||||
|
||||
**Alerting hooks:** Emit structured logs/metrics when either the limiter or credential store rejects access. Suggested gauges include `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` and any custom `auth.plugins.<pluginName>.lockouts_total` counter.
|
||||
|
||||
|
||||
|
||||
_Source:_ `docs/assets/authority/authority-rate-limit-flow.mmd`
|
||||
|
||||
## 9. Logging, Metrics, and Diagnostics
|
||||
- Always log via the injected `ILogger<T>`; include `pluginName` and correlation IDs where available.
|
||||
- Activity/metric names should align with `AuthorityTelemetry` constants (`service.name=stellaops-authority`).
|
||||
- Expose additional diagnostics via structured logging rather than writing custom HTTP endpoints; the host will integrate these into `/health` and `/ready`.
|
||||
- Emit metrics with stable names (`auth.plugins.<pluginName>.*`) when introducing custom instrumentation; coordinate with the Observability guild to reserve prefixes.
|
||||
|
||||
## 10. Testing & Tooling
|
||||
- Unit tests: use Mongo2Go (or similar) to exercise credential stores without hitting production infrastructure (`StandardUserCredentialStoreTests` is a template).
|
||||
- Determinism: fix timestamps to UTC and sort outputs consistently; avoid random GUIDs unless stable.
|
||||
- Smoke tests: launch `dotnet run --project src/StellaOps.Authority/StellaOps.Authority` with your plug-in under `StellaOps.Authority.PluginBinaries` and verify `/ready`.
|
||||
- Example verification snippet:
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task VerifyPasswordAsync_ReturnsSuccess()
|
||||
{
|
||||
var store = CreateCredentialStore();
|
||||
await store.UpsertUserAsync(new AuthorityUserRegistration("alice", "Pa55!", null, null, false,
|
||||
Array.Empty<string>(), new Dictionary<string, string?>()), CancellationToken.None);
|
||||
|
||||
var result = await store.VerifyPasswordAsync("alice", "Pa55!", CancellationToken.None);
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.True(result.User?.Roles.Count == 0);
|
||||
}
|
||||
```
|
||||
|
||||
## 11. Packaging & Delivery
|
||||
- Output assembly should follow `StellaOps.Authority.Plugin.<Name>.dll` so the host’s search pattern picks it up.
|
||||
- Place the compiled DLL plus dependencies under `StellaOps.Authority.PluginBinaries` for offline deployments; include hashes/signatures in release notes (Security Guild guidance forthcoming).
|
||||
- Document any external prerequisites (e.g., CA cert bundle) in your plug-in README.
|
||||
- Update `etc/authority.plugins/<plugin>.yaml` samples and include deterministic SHA256 hashes for optional bootstrap payloads when distributing Offline Kit artefacts.
|
||||
|
||||
[^ldap-rfc]: Lightweight Directory Access Protocol (LDAPv3) specification — [RFC 4511](https://datatracker.ietf.org/doc/html/rfc4511).
|
||||
|
||||
## 12. Checklist & Handoff
|
||||
- ✅ Capabilities declared and validated in automated tests.
|
||||
- ✅ Bootstrap workflows documented (if `bootstrap` capability used) and repeatable.
|
||||
- ✅ Local smoke test + unit/integration suites green (`dotnet test`).
|
||||
- ✅ Operational docs updated: configuration keys, secrets guidance, troubleshooting.
|
||||
- Submit the developer guide update referencing PLG6/DOC4 and tag DevEx + Docs reviewers for sign-off.
|
||||
|
||||
---
|
||||
Mermaid sources for the embedded diagrams live under `docs/assets/authority/`. Regenerate the SVG assets with your preferred renderer before committing future updates so the visuals stay in sync with the `.mmd` definitions.
|
||||
|
||||
@@ -1,91 +1,91 @@
|
||||
# StellaOps Auth Client — Integration Guide
|
||||
|
||||
> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Feedser, CLI, Agent) should review before wiring the new options into their configuration surfaces.
|
||||
|
||||
The `StellaOps.Auth.Client` library provides a resilient OpenID Connect client for services and tools that talk to **StellaOps Authority**. LIB5 introduced configurable HTTP retry/backoff policies and an offline-fallback window so downstream components stay deterministic even when Authority is briefly unavailable.
|
||||
|
||||
This guide explains how to consume the new settings, when to toggle them, and how to test your integration.
|
||||
|
||||
## 1. Registering the client
|
||||
|
||||
```csharp
|
||||
services.AddStellaOpsAuthClient(options =>
|
||||
{
|
||||
options.Authority = configuration["StellaOps:Authority:Url"]!;
|
||||
options.ClientId = configuration["StellaOps:Authority:ClientId"]!;
|
||||
options.ClientSecret = configuration["StellaOps:Authority:ClientSecret"];
|
||||
options.DefaultScopes.Add("feedser.jobs.trigger");
|
||||
|
||||
options.EnableRetries = true;
|
||||
options.RetryDelays.Clear();
|
||||
options.RetryDelays.Add(TimeSpan.FromMilliseconds(500));
|
||||
options.RetryDelays.Add(TimeSpan.FromSeconds(2));
|
||||
|
||||
options.AllowOfflineCacheFallback = true;
|
||||
options.OfflineCacheTolerance = TimeSpan.FromMinutes(5);
|
||||
});
|
||||
```
|
||||
|
||||
> **Reminder:** `AddStellaOpsAuthClient` binds the options via `IOptionsMonitor<T>` so changes picked up from configuration reloads will be applied to future HTTP calls without restarting the host.
|
||||
|
||||
## 2. Resilience options
|
||||
|
||||
| Option | Default | Notes |
|
||||
|--------|---------|-------|
|
||||
| `EnableRetries` | `true` | When disabled, the shared Polly policy is a no-op and HTTP calls will fail fast. |
|
||||
| `RetryDelays` | `1s, 2s, 5s` | Edit in ascending order; zero/negative entries are ignored. Clearing the list and leaving it empty keeps the defaults. |
|
||||
| `AllowOfflineCacheFallback` | `true` | When `true`, stale discovery/JWKS responses are reused within the tolerance window if Authority is unreachable. |
|
||||
| `OfflineCacheTolerance` | `00:10:00` | Added to the normal cache lifetime. E.g. a 10 minute JWKS cache plus 5 minute tolerance keeps keys for 15 minutes if Authority is offline. |
|
||||
|
||||
The HTTP retry policy handles:
|
||||
|
||||
- 5xx responses
|
||||
- 429 responses
|
||||
- Transient transport failures (`HttpRequestException`, timeouts, aborted sockets)
|
||||
|
||||
Retries emit warnings via the `StellaOps.Auth.Client.HttpRetry` logger. Tune the delay values to honour your deployment’s SLOs.
|
||||
|
||||
## 3. Configuration mapping
|
||||
|
||||
Suggested configuration keys (coordinate with consuming teams before finalising):
|
||||
|
||||
```yaml
|
||||
StellaOps:
|
||||
Authority:
|
||||
Url: "https://authority.stella-ops.local"
|
||||
ClientId: "feedser"
|
||||
ClientSecret: "change-me"
|
||||
AuthClient:
|
||||
EnableRetries: true
|
||||
RetryDelays:
|
||||
- "00:00:01"
|
||||
- "00:00:02"
|
||||
- "00:00:05"
|
||||
AllowOfflineCacheFallback: true
|
||||
OfflineCacheTolerance: "00:10:00"
|
||||
```
|
||||
|
||||
Environment variable binding follows the usual double-underscore rules, e.g.
|
||||
|
||||
```
|
||||
STELLAOPS__AUTHORITY__AUTHCLIENT__RETRYDELAYS__0=00:00:02
|
||||
STELLAOPS__AUTHORITY__AUTHCLIENT__OFFLINECACHETOLERANCE=00:05:00
|
||||
```
|
||||
|
||||
CLI and Feedser teams should expose these knobs once they adopt the auth client.
|
||||
|
||||
## 4. Testing recommendations
|
||||
|
||||
1. **Unit tests:** assert option binding by configuring `StellaOpsAuthClientOptions` via a `ConfigurationBuilder` and ensuring `Validate()` normalises the retry delays and scope list.
|
||||
2. **Offline fallback:** simulate an unreachable Authority by swapping `HttpMessageHandler` to throw `HttpRequestException` after priming the discovery/JWKS caches. Verify that tokens are still issued until the tolerance expires.
|
||||
3. **Observability:** watch for `StellaOps.Auth.Client.HttpRetry` warnings in your logs. Excessive retries mean the upstream Authority cluster needs attention.
|
||||
4. **Determinism:** keep retry delays deterministic. Avoid random jitter—operators can introduce jitter at the infrastructure layer if desired.
|
||||
|
||||
## 5. Rollout checklist
|
||||
|
||||
- [ ] Update consuming service/CLI configuration schema to include the new settings.
|
||||
- [ ] Document recommended defaults for offline (air-gapped) versus connected deployments.
|
||||
- [ ] Extend smoke tests to cover Authority outage scenarios.
|
||||
- [ ] Coordinate with Docs Guild so user-facing quickstarts reference the new knobs.
|
||||
|
||||
Once Feedser and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it.
|
||||
# StellaOps Auth Client — Integration Guide
|
||||
|
||||
> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Concelier, CLI, Agent) should review before wiring the new options into their configuration surfaces.
|
||||
|
||||
The `StellaOps.Auth.Client` library provides a resilient OpenID Connect client for services and tools that talk to **StellaOps Authority**. LIB5 introduced configurable HTTP retry/backoff policies and an offline-fallback window so downstream components stay deterministic even when Authority is briefly unavailable.
|
||||
|
||||
This guide explains how to consume the new settings, when to toggle them, and how to test your integration.
|
||||
|
||||
## 1. Registering the client
|
||||
|
||||
```csharp
|
||||
services.AddStellaOpsAuthClient(options =>
|
||||
{
|
||||
options.Authority = configuration["StellaOps:Authority:Url"]!;
|
||||
options.ClientId = configuration["StellaOps:Authority:ClientId"]!;
|
||||
options.ClientSecret = configuration["StellaOps:Authority:ClientSecret"];
|
||||
options.DefaultScopes.Add("concelier.jobs.trigger");
|
||||
|
||||
options.EnableRetries = true;
|
||||
options.RetryDelays.Clear();
|
||||
options.RetryDelays.Add(TimeSpan.FromMilliseconds(500));
|
||||
options.RetryDelays.Add(TimeSpan.FromSeconds(2));
|
||||
|
||||
options.AllowOfflineCacheFallback = true;
|
||||
options.OfflineCacheTolerance = TimeSpan.FromMinutes(5);
|
||||
});
|
||||
```
|
||||
|
||||
> **Reminder:** `AddStellaOpsAuthClient` binds the options via `IOptionsMonitor<T>` so changes picked up from configuration reloads will be applied to future HTTP calls without restarting the host.
|
||||
|
||||
## 2. Resilience options
|
||||
|
||||
| Option | Default | Notes |
|
||||
|--------|---------|-------|
|
||||
| `EnableRetries` | `true` | When disabled, the shared Polly policy is a no-op and HTTP calls will fail fast. |
|
||||
| `RetryDelays` | `1s, 2s, 5s` | Edit in ascending order; zero/negative entries are ignored. Clearing the list and leaving it empty keeps the defaults. |
|
||||
| `AllowOfflineCacheFallback` | `true` | When `true`, stale discovery/JWKS responses are reused within the tolerance window if Authority is unreachable. |
|
||||
| `OfflineCacheTolerance` | `00:10:00` | Added to the normal cache lifetime. E.g. a 10 minute JWKS cache plus 5 minute tolerance keeps keys for 15 minutes if Authority is offline. |
|
||||
|
||||
The HTTP retry policy handles:
|
||||
|
||||
- 5xx responses
|
||||
- 429 responses
|
||||
- Transient transport failures (`HttpRequestException`, timeouts, aborted sockets)
|
||||
|
||||
Retries emit warnings via the `StellaOps.Auth.Client.HttpRetry` logger. Tune the delay values to honour your deployment’s SLOs.
|
||||
|
||||
## 3. Configuration mapping
|
||||
|
||||
Suggested configuration keys (coordinate with consuming teams before finalising):
|
||||
|
||||
```yaml
|
||||
StellaOps:
|
||||
Authority:
|
||||
Url: "https://authority.stella-ops.local"
|
||||
ClientId: "concelier"
|
||||
ClientSecret: "change-me"
|
||||
AuthClient:
|
||||
EnableRetries: true
|
||||
RetryDelays:
|
||||
- "00:00:01"
|
||||
- "00:00:02"
|
||||
- "00:00:05"
|
||||
AllowOfflineCacheFallback: true
|
||||
OfflineCacheTolerance: "00:10:00"
|
||||
```
|
||||
|
||||
Environment variable binding follows the usual double-underscore rules, e.g.
|
||||
|
||||
```
|
||||
STELLAOPS__AUTHORITY__AUTHCLIENT__RETRYDELAYS__0=00:00:02
|
||||
STELLAOPS__AUTHORITY__AUTHCLIENT__OFFLINECACHETOLERANCE=00:05:00
|
||||
```
|
||||
|
||||
CLI and Concelier teams should expose these knobs once they adopt the auth client.
|
||||
|
||||
## 4. Testing recommendations
|
||||
|
||||
1. **Unit tests:** assert option binding by configuring `StellaOpsAuthClientOptions` via a `ConfigurationBuilder` and ensuring `Validate()` normalises the retry delays and scope list.
|
||||
2. **Offline fallback:** simulate an unreachable Authority by swapping `HttpMessageHandler` to throw `HttpRequestException` after priming the discovery/JWKS caches. Verify that tokens are still issued until the tolerance expires.
|
||||
3. **Observability:** watch for `StellaOps.Auth.Client.HttpRetry` warnings in your logs. Excessive retries mean the upstream Authority cluster needs attention.
|
||||
4. **Determinism:** keep retry delays deterministic. Avoid random jitter—operators can introduce jitter at the infrastructure layer if desired.
|
||||
|
||||
## 5. Rollout checklist
|
||||
|
||||
- [ ] Update consuming service/CLI configuration schema to include the new settings.
|
||||
- [ ] Document recommended defaults for offline (air-gapped) versus connected deployments.
|
||||
- [ ] Extend smoke tests to cover Authority outage scenarios.
|
||||
- [ ] Coordinate with Docs Guild so user-facing quickstarts reference the new knobs.
|
||||
|
||||
Once Concelier and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it.
|
||||
|
||||
119
docs/dev/BUILDX_PLUGIN_QUICKSTART.md
Normal file
119
docs/dev/BUILDX_PLUGIN_QUICKSTART.md
Normal file
@@ -0,0 +1,119 @@
|
||||
# BuildX Generator Quickstart
|
||||
|
||||
This quickstart explains how to run the StellaOps **BuildX SBOM generator** offline, verify the CAS handshake, and emit OCI descriptors that downstream services can attest.
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
- Docker 25+ with BuildKit enabled (`docker buildx` available).
|
||||
- .NET 10 (preview) SDK matching the repository `global.json`.
|
||||
- Optional: network access to a StellaOps Attestor endpoint (the quickstart uses a mock service).
|
||||
|
||||
## 2. Publish the plug-in binaries
|
||||
|
||||
The BuildX generator publishes as a .NET self-contained executable with its manifest under `plugins/scanner/buildx/`.
|
||||
|
||||
```bash
|
||||
# From the repository root
|
||||
DOTNET_CLI_HOME="${PWD}/.dotnet" \
|
||||
dotnet publish src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj \
|
||||
-c Release \
|
||||
-o out/buildx
|
||||
```
|
||||
|
||||
- `out/buildx/` now contains `StellaOps.Scanner.Sbomer.BuildXPlugin.dll` and the manifest `stellaops.sbom-indexer.manifest.json`.
|
||||
- `plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/` receives the same artefacts for release packaging.
|
||||
- The CI pipeline also tars and signs (SHA-256 manifest) the OS analyzer plug-ins located under
|
||||
`plugins/scanner/analyzers/os/` so they ship alongside the BuildX generator artefacts.
|
||||
|
||||
## 3. Verify the CAS handshake
|
||||
|
||||
```bash
|
||||
dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll handshake \
|
||||
--manifest out/buildx \
|
||||
--cas out/cas
|
||||
```
|
||||
|
||||
The command performs a deterministic probe write (`sha256`) into the provided CAS directory and prints the resolved path.
|
||||
|
||||
## 4. Emit a descriptor + provenance placeholder
|
||||
|
||||
1. Build or identify the image you want to describe and capture its digest:
|
||||
|
||||
```bash
|
||||
docker buildx build --load -t stellaops/buildx-demo:ci samples/ci/buildx-demo
|
||||
DIGEST=$(docker image inspect stellaops/buildx-demo:ci --format '{{index .RepoDigests 0}}')
|
||||
```
|
||||
|
||||
2. Generate a CycloneDX SBOM for the built image (any tool works; here we use `docker sbom`):
|
||||
|
||||
```bash
|
||||
docker sbom stellaops/buildx-demo:ci --format cyclonedx-json > out/buildx-sbom.cdx.json
|
||||
```
|
||||
|
||||
3. Invoke the `descriptor` command, pointing at the SBOM file and optional metadata:
|
||||
|
||||
```bash
|
||||
dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \
|
||||
--manifest out/buildx \
|
||||
--image "$DIGEST" \
|
||||
--sbom out/buildx-sbom.cdx.json \
|
||||
--sbom-name buildx-sbom.cdx.json \
|
||||
--artifact-type application/vnd.stellaops.sbom.layer+json \
|
||||
--sbom-format cyclonedx-json \
|
||||
--sbom-kind inventory \
|
||||
--repository git.stella-ops.org/stellaops/buildx-demo \
|
||||
--build-ref $(git rev-parse HEAD) \
|
||||
> out/buildx-descriptor.json
|
||||
```
|
||||
|
||||
The output JSON captures:
|
||||
|
||||
- OCI artifact descriptor including size, digest, and annotations (`org.stellaops.*`).
|
||||
- Provenance placeholder (`expectedDsseSha256`, `nonce`, `attestorUri` when provided). `nonce` is derived deterministically from the image + SBOM metadata so repeated runs produce identical placeholders for identical inputs.
|
||||
- Generator metadata and deterministic timestamps.
|
||||
|
||||
## 5. (Optional) Send the placeholder to an Attestor
|
||||
|
||||
The plug-in can POST the descriptor metadata to an Attestor endpoint, returning once it receives an HTTP 202.
|
||||
|
||||
```bash
|
||||
python3 - <<'PY' &
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
_ = self.rfile.read(int(self.headers.get('Content-Length', 0)))
|
||||
self.send_response(202); self.end_headers(); self.wfile.write(b'accepted')
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
server = HTTPServer(('127.0.0.1', 8085), Handler)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
PY
|
||||
MOCK_PID=$!
|
||||
|
||||
dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \
|
||||
--manifest out/buildx \
|
||||
--image "$DIGEST" \
|
||||
--sbom out/buildx-sbom.cdx.json \
|
||||
--attestor http://127.0.0.1:8085/provenance \
|
||||
--attestor-token "$STELLAOPS_ATTESTOR_TOKEN" \
|
||||
> out/buildx-descriptor.json
|
||||
|
||||
kill $MOCK_PID
|
||||
```
|
||||
|
||||
Set `STELLAOPS_ATTESTOR_TOKEN` (or pass `--attestor-token`) when the Attestor requires bearer authentication. Use `--attestor-insecure` for lab environments with self-signed certificates.
|
||||
|
||||
## 6. CI workflow example
|
||||
|
||||
A reusable GitHub Actions workflow is provided under `samples/ci/buildx-demo/github-actions-buildx-demo.yml`. It publishes the plug-in, runs the handshake, builds the demo image, emits a descriptor, and uploads both the descriptor and the mock-Attestor request as artefacts.
|
||||
|
||||
Add the workflow to your repository (or call it via `workflow_call`) and adjust the SBOM path + Attestor URL as needed. The workflow also re-runs the `descriptor` command and diffs the results (ignoring the `generatedAt` timestamp) so you catch regressions that would break deterministic CI use.
|
||||
|
||||
---
|
||||
|
||||
For deeper integration guidance (custom SBOM builders, exporting DSSE bundles), track ADRs in `docs/ARCHITECTURE_SCANNER.md` §7 and follow upcoming Attestor API releases.
|
||||
86
docs/dev/EXCITITOR_STATEMENT_BACKFILL.md
Normal file
86
docs/dev/EXCITITOR_STATEMENT_BACKFILL.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# Excititor Statement Backfill Runbook
|
||||
|
||||
Last updated: 2025-10-19
|
||||
|
||||
## Overview
|
||||
|
||||
Use this runbook when you need to rebuild the `vex.statements` collection from historical raw documents. Typical scenarios:
|
||||
|
||||
- Upgrading the statement schema (e.g., adding severity/KEV/EPSS signals).
|
||||
- Recovering from a partial ingest outage where statements were never persisted.
|
||||
- Seeding a freshly provisioned Excititor deployment from an existing raw archive.
|
||||
|
||||
Backfill operates server-side via the Excititor WebService and reuses the same pipeline that powers the `/excititor/statements` ingestion endpoint. Each raw document is normalized, signed metadata is preserved, and duplicate statements are skipped unless the run is forced.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Connectivity to Excititor WebService** – the CLI uses the backend URL configured in `stellaops.yml` or the `--backend-url` argument.
|
||||
2. **Authority credentials** – the CLI honours the existing Authority client configuration; ensure the caller has permission to invoke admin endpoints.
|
||||
3. **Mongo replica set** (recommended) – causal consistency guarantees rely on majority read/write concerns. Standalone deployment works but skips cross-document transactions.
|
||||
|
||||
## CLI command
|
||||
|
||||
```
|
||||
stellaops excititor backfill-statements \
|
||||
[--retrieved-since <ISO8601>] \
|
||||
[--force] \
|
||||
[--batch-size <int>] \
|
||||
[--max-documents <int>]
|
||||
```
|
||||
|
||||
| Option | Description |
|
||||
| ------ | ----------- |
|
||||
| `--retrieved-since` | Only process raw documents fetched on or after the specified timestamp (UTC by default). |
|
||||
| `--force` | Reprocess documents even if matching statements already exist (useful after schema upgrades). |
|
||||
| `--batch-size` | Number of raw documents pulled per batch (default `100`). |
|
||||
| `--max-documents` | Optional hard limit on the number of raw documents to evaluate. |
|
||||
|
||||
Example – replay the last 48 hours of Red Hat ingest while keeping existing statements:
|
||||
|
||||
```
|
||||
stellaops excititor backfill-statements \
|
||||
--retrieved-since "$(date -u -d '48 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
|
||||
```
|
||||
|
||||
Example – full replay with forced overwrites, capped at 2,000 documents:
|
||||
|
||||
```
|
||||
stellaops excititor backfill-statements --force --max-documents 2000
|
||||
```
|
||||
|
||||
The command returns a summary similar to:
|
||||
|
||||
```
|
||||
Backfill completed: evaluated 450, backfilled 180, claims written 320, skipped 270, failures 0.
|
||||
```
|
||||
|
||||
## Behaviour
|
||||
|
||||
- Raw documents are streamed in ascending `retrievedAt` order.
|
||||
- Each document is normalized using the registered VEX normalizers (CSAF, CycloneDX, OpenVEX).
|
||||
- Statements are appended through the same `IVexClaimStore.AppendAsync` path that powers `/excititor/statements`.
|
||||
- Duplicate detection compares `Document.Digest`; duplicates are skipped unless `--force` is specified.
|
||||
- Failures are logged with the offending digest and continue with the next document.
|
||||
|
||||
## Observability
|
||||
|
||||
- CLI logs aggregate counts and the backend logs per-digest warnings or errors.
|
||||
- Mongo writes carry majority write concern; expect backfill throughput to match ingest baselines (≈5 seconds warm, 30 seconds cold).
|
||||
- Monitor the `excititor.storage.backfill` log scope for detailed telemetry.
|
||||
|
||||
## Post-run verification
|
||||
|
||||
1. Inspect the `vex.statements` collection for the targeted window (check `InsertedAt`).
|
||||
2. Re-run the Excititor storage test suite if possible:
|
||||
```
|
||||
dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj
|
||||
```
|
||||
3. Optionally, call `/excititor/statements/{vulnerabilityId}/{productKey}` to confirm the expected statements exist.
|
||||
|
||||
## Rollback
|
||||
|
||||
If a forced run produced incorrect statements, use the standard Mongo rollback procedure:
|
||||
|
||||
1. Identify the `InsertedAt` window for the backfill run.
|
||||
2. Delete affected records from `vex.statements` (and any downstream exports if applicable).
|
||||
3. Rerun the backfill command with corrected parameters.
|
||||
108
docs/dev/SCANNER_CACHE_CONFIGURATION.md
Normal file
108
docs/dev/SCANNER_CACHE_CONFIGURATION.md
Normal file
@@ -0,0 +1,108 @@
|
||||
# Scanner Cache Configuration Guide
|
||||
|
||||
The scanner cache stores layer-level SBOM fragments and file content that can be reused across scans. This document explains how to configure and operate the cache subsystem introduced in Sprint 10 (Group SP10-G5).
|
||||
|
||||
## 1. Overview
|
||||
|
||||
- **Layer cache** persists SBOM fragments per layer digest under `<root>/layers/<digest>/` with deterministic metadata (`meta.json`).
|
||||
- **File CAS** (content-addressable store) keeps deduplicated blobs (e.g., analyzer fixtures, imported SBOM layers) under `<root>/cas/<prefix>/<hash>/`.
|
||||
- **Maintenance** runs via `ScannerCacheMaintenanceService`, evicting expired entries and compacting the cache to stay within size limits.
|
||||
- **Metrics** emit on the `StellaOps.Scanner.Cache` meter with counters for hits, misses, evictions, and byte histograms.
|
||||
- **Offline workflows** use the CAS import/export helpers to package cache warmups inside the Offline Kit.
|
||||
|
||||
## 2. Configuration keys (`scanner:cache`)
|
||||
|
||||
| Key | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `enabled` | `true` | Globally disable cache if `false`. |
|
||||
| `rootPath` | `cache/scanner` | Base directory for cache data. Use an SSD-backed path for best warm-scan latency. |
|
||||
| `layersDirectoryName` | `layers` | Subdirectory for layer cache entries. |
|
||||
| `fileCasDirectoryName` | `cas` | Subdirectory for file CAS entries. |
|
||||
| `layerTtl` | `45.00:00:00` | Time-to-live for layer cache entries (`TimeSpan`). `0` disables TTL eviction. |
|
||||
| `fileTtl` | `30.00:00:00` | Time-to-live for CAS entries. `0` disables TTL eviction. |
|
||||
| `maxBytes` | `5368709120` (5 GiB) | Hard cap for combined cache footprint. Compaction trims data back to `warmBytesThreshold`. |
|
||||
| `warmBytesThreshold` | `maxBytes / 5` | Target size after compaction. |
|
||||
| `coldBytesThreshold` | `maxBytes * 0.8` | Upper bound that triggers compaction. |
|
||||
| `enableAutoEviction` | `true` | If `false`, callers must invoke `ILayerCacheStore.CompactAsync` / `IFileContentAddressableStore.CompactAsync` manually. |
|
||||
| `maintenanceInterval` | `00:15:00` | Interval for the maintenance hosted service. |
|
||||
| `enableFileCas` | `true` | Disable to prevent CAS usage (APIs throw on `PutAsync`). |
|
||||
| `importDirectory` / `exportDirectory` | `null` | Optional defaults for offline import/export tooling. |
|
||||
|
||||
> **Tip:** configure `scanner:cache:rootPath` to a dedicated volume and mount it into worker containers when running in Kubernetes or Nomad.
|
||||
|
||||
## 3. Metrics
|
||||
|
||||
Instrumentation lives in `ScannerCacheMetrics` on meter `StellaOps.Scanner.Cache`.
|
||||
|
||||
| Instrument | Unit | Description |
|
||||
| --- | --- | --- |
|
||||
| `scanner.layer_cache_hits_total` | count | Layer cache hit counter. Tag: `layer`. |
|
||||
| `scanner.layer_cache_misses_total` | count | Layer cache miss counter. Tag: `layer`. |
|
||||
| `scanner.layer_cache_evictions_total` | count | Layer entries evicted due to TTL or compaction. Tag: `layer`. |
|
||||
| `scanner.layer_cache_bytes` | bytes | Histogram of per-entry payload size when stored. |
|
||||
| `scanner.file_cas_hits_total` | count | File CAS hit counter. Tag: `sha256`. |
|
||||
| `scanner.file_cas_misses_total` | count | File CAS miss counter. Tag: `sha256`. |
|
||||
| `scanner.file_cas_evictions_total` | count | CAS eviction counter. Tag: `sha256`. |
|
||||
| `scanner.file_cas_bytes` | bytes | Histogram of CAS payload sizes on insert. |
|
||||
|
||||
## 4. Import / Export workflow
|
||||
|
||||
1. **Export warm cache**
|
||||
```bash
|
||||
dotnet tool run stellaops-cache export --destination ./offline-kit/cache
|
||||
```
|
||||
Internally this calls `IFileContentAddressableStore.ExportAsync` which copies each CAS entry (metadata + `content.bin`).
|
||||
|
||||
2. **Import on air-gapped hosts**
|
||||
```bash
|
||||
dotnet tool run stellaops-cache import --source ./offline-kit/cache
|
||||
```
|
||||
The import API merges newer metadata and skips older snapshots automatically.
|
||||
|
||||
3. **Layer cache seeding**
|
||||
Layer cache entries are deterministic and can be packaged the same way (copy `<root>/layers`). For now we keep seeding optional because layers are larger; follow-up tooling can compress directories as needed.
|
||||
|
||||
## 5. Hosted maintenance loop
|
||||
|
||||
`ScannerCacheMaintenanceService` runs as a background service within Scanner Worker or WebService hosts when `AddScannerCache` is registered. Behaviour:
|
||||
|
||||
- At startup it performs an immediate eviction/compaction run.
|
||||
- Every `maintenanceInterval` it triggers:
|
||||
- `ILayerCacheStore.EvictExpiredAsync`
|
||||
- `ILayerCacheStore.CompactAsync`
|
||||
- `IFileContentAddressableStore.EvictExpiredAsync`
|
||||
- `IFileContentAddressableStore.CompactAsync`
|
||||
- Failures are logged at `Error` with preserved stack traces; the next tick continues normally.
|
||||
|
||||
Set `enableAutoEviction=false` when hosting the cache inside ephemeral build pipelines that want to drive eviction explicitly.
|
||||
|
||||
## 6. API surface summary
|
||||
|
||||
```csharp
|
||||
public interface ILayerCacheStore
|
||||
{
|
||||
ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken ct = default);
|
||||
Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken ct = default);
|
||||
Task RemoveAsync(string layerDigest, CancellationToken ct = default);
|
||||
Task<int> EvictExpiredAsync(CancellationToken ct = default);
|
||||
Task<int> CompactAsync(CancellationToken ct = default);
|
||||
Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
public interface IFileContentAddressableStore
|
||||
{
|
||||
ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken ct = default);
|
||||
Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken ct = default);
|
||||
Task<bool> RemoveAsync(string sha256, CancellationToken ct = default);
|
||||
Task<int> EvictExpiredAsync(CancellationToken ct = default);
|
||||
Task<int> CompactAsync(CancellationToken ct = default);
|
||||
Task<int> ExportAsync(string destinationDirectory, CancellationToken ct = default);
|
||||
Task<int> ImportAsync(string sourceDirectory, CancellationToken ct = default);
|
||||
}
|
||||
```
|
||||
|
||||
Register both stores via `services.AddScannerCache(configuration);` in WebService or Worker hosts.
|
||||
|
||||
---
|
||||
|
||||
_Last updated: 2025-10-19_
|
||||
142
docs/dev/authority-dpop-mtls-plan.md
Normal file
142
docs/dev/authority-dpop-mtls-plan.md
Normal file
@@ -0,0 +1,142 @@
|
||||
# Authority DPoP & mTLS Implementation Plan (2025-10-19)
|
||||
|
||||
## Purpose
|
||||
- Provide the implementation blueprint for AUTH-DPOP-11-001 and AUTH-MTLS-11-002.
|
||||
- Unify sender-constraint validation across Authority, downstream services, and clients.
|
||||
- Capture deterministic, testable steps that unblock UI/Signer guilds depending on DPoP/mTLS hardening.
|
||||
|
||||
## Scope
|
||||
- Token endpoint validation, issuance, and storage changes inside `StellaOps.Authority`.
|
||||
- Shared security primitives consumed by Authority, Scanner, Signer, CLI, and UI.
|
||||
- Operator-facing configuration, auditing, and observability.
|
||||
- Out of scope: PoE enforcement (Signer) and CLI/UI client UX; those teams consume the new capabilities.
|
||||
|
||||
> **Status update (2025-10-19):** `ValidateDpopProofHandler`, `AuthorityClientCertificateValidator`, and the supporting storage/audit plumbing now live in `src/StellaOps.Authority`. DPoP proofs populate `cnf.jkt`, mTLS bindings enforce certificate thumbprints via `cnf.x5t#S256`, and token documents persist the sender constraint metadata. In-memory nonce issuance is wired (Redis implementation to follow). Documentation and configuration references were updated (`docs/11_AUTHORITY.md`). Targeted unit/integration tests were added; running the broader test suite is currently blocked by pre-existing `StellaOps.Concelier.Storage.Mongo` build errors.
|
||||
|
||||
## Design Summary
|
||||
- Extract the existing Scanner `DpopProofValidator` stack into a shared `StellaOps.Auth.Security` library used by Authority and resource servers.
|
||||
- Extend Authority configuration (`authority.yaml`) with strongly-typed `senderConstraints.dpop` and `senderConstraints.mtls` sections (map to sample already shown in architecture doc).
|
||||
- Require DPoP proofs on `/token` when the registered client policy is `senderConstraint=dpop`; bind issued access tokens via `cnf.jkt`.
|
||||
- Introduce Authority-managed nonce issuance for “high value” audiences (default: `signer`, `attestor`) with Redis-backed persistence and deterministic auditing.
|
||||
- Enable OAuth 2.0 mTLS (RFC 8705) by storing certificate bindings per client, requesting client certificates at TLS termination, and stamping `cnf.x5t#S256` into issued tokens plus introspection output.
|
||||
- Surface structured logs and counters for both DPoP and mTLS flows; provide integration tests that cover success, replay, invalid proof, and certificate mismatch cases.
|
||||
|
||||
## AUTH-DPOP-11-001 — Proof Validation & Nonce Handling
|
||||
|
||||
**Shared validator**
|
||||
- Move `DpopProofValidator`, option types, and replay cache interfaces from `StellaOps.Scanner.Core` into a new assembly `StellaOps.Auth.Security`.
|
||||
- Provide pluggable caches: `InMemoryDpopReplayCache` (existing) and new `RedisDpopReplayCache` (leveraging the Authority Redis connection).
|
||||
- Ensure the validator exposes the validated `SecurityKey`, `jti`, and `iat` so Authority can construct the `cnf` claim and compute nonce expiry.
|
||||
|
||||
**Configuration model**
|
||||
- Extend `StellaOpsAuthorityOptions.Security` with a `SenderConstraints` property containing:
|
||||
- `Dpop` (`enabled`, `allowedAlgorithms`, `maxAgeSeconds`, `clockSkewSeconds`, `replayWindowSeconds`, `nonce` settings with `enabled`, `ttlSeconds`, `requiredAudiences`, `maxIssuancePerMinute`).
|
||||
- `Mtls` (`enabled`, `requireChainValidation`, `clientCaBundle`, `allowedSubjectPatterns`, `allowedSanTypes`).
|
||||
- Bind from YAML (`authority.security.senderConstraints.*`) while preserving backwards compatibility (defaults keep both disabled).
|
||||
|
||||
**Token endpoint pipeline**
|
||||
- Introduce a scoped OpenIddict handler `ValidateDpopProofHandler` inserted before `ValidateClientCredentialsHandler`.
|
||||
- Determine the required sender constraint from client metadata:
|
||||
- Add `AuthorityClientMetadataKeys.SenderConstraint` storing `dpop` or `mtls`.
|
||||
- Optionally allow per-client overrides for nonce requirement.
|
||||
- When `dpop` is required:
|
||||
- Read the `DPoP` header from the ASP.NET request, reject with `invalid_token` + `WWW-Authenticate: DPoP error="invalid_dpop_proof"` if absent.
|
||||
- Call the shared validator with method/URI. Enforce algorithm allowlist and `iat` window from options.
|
||||
- Persist the `jkt` thumbprint plus replay cache state in the OpenIddict transaction (`AuthorityOpenIddictConstants.DpopKeyThumbprintProperty`, `DpopIssuedAtProperty`).
|
||||
- When the requested audience intersects `SenderConstraints.Dpop.Nonce.RequiredAudiences`, require `nonce` in the proof; on first failure respond with HTTP 401, `error="use_dpop_nonce"`, and include `DPoP-Nonce` header (see nonce note below). Cache the rejection reason for audit logging.
|
||||
|
||||
**Nonce service**
|
||||
- Add `IDpopNonceStore` with methods `IssueAsync(audience, clientId, jkt)` and `TryConsumeAsync(nonce, audience, clientId, jkt)`.
|
||||
- Default implementation `RedisDpopNonceStore` storing SHA-256 hashes of nonces keyed by `audience:clientId:jkt`. TTL comes from `SenderConstraints.Dpop.Nonce.Ttl`.
|
||||
- Create helper `DpopNonceIssuer` used by `ValidateDpopProofHandler` to issue nonces when missing/expired, enforcing issuance rate limits (per options) and tagging audit/log records.
|
||||
- On successful validation (nonce supplied and consumed) stamp metadata into the transaction for auditing.
|
||||
- Update `ClientCredentialsHandlers` to observe nonce enforcement: when a nonce challenge was sent, emit structured audit with `nonce_issued`, `audiences`, and `retry`.
|
||||
|
||||
**Token issuance**
|
||||
- In `HandleClientCredentialsHandler`, if the transaction contains a validated DPoP key:
|
||||
- Build `cnf.jkt` using thumbprint from validator.
|
||||
- Include `auth_time`/`dpop_jti` as needed for diagnostics.
|
||||
- Persist the thumbprint alongside token metadata in Mongo (extend `AuthorityTokenDocument` with `SenderConstraint`, `KeyThumbprint`, `Nonce` fields).
|
||||
|
||||
**Auditing & observability**
|
||||
- Emit new audit events:
|
||||
- `authority.dpop.proof.validated` (success/failure, clientId, audience, thumbprint, nonce status, jti).
|
||||
- `authority.dpop.nonce.issued` and `authority.dpop.nonce.consumed`.
|
||||
- Metrics (Prometheus style):
|
||||
- `authority_dpop_validations_total{result,reason}`.
|
||||
- `authority_dpop_nonce_issued_total{audience}` and `authority_dpop_nonce_fails_total{reason}`.
|
||||
- Structured logs include `authority.sender_constraint=dpop`, `authority.dpop_thumbprint`, `authority.dpop_nonce`.
|
||||
|
||||
**Testing**
|
||||
- Unit tests for the handler pipeline using fake OpenIddict transactions.
|
||||
- Replay/nonce tests with in-memory and Redis stores.
|
||||
- Integration tests in `StellaOps.Authority.Tests` covering:
|
||||
- Valid DPoP proof issuing `cnf.jkt`.
|
||||
- Missing header → challenge with nonce.
|
||||
- Replayed `jti` rejected.
|
||||
- Invalid nonce rejected even after issuance.
|
||||
- Contract tests to ensure `/.well-known/openid-configuration` advertises `dpop_signing_alg_values_supported` and `dpop_nonce_supported` when enabled.
|
||||
|
||||
## AUTH-MTLS-11-002 — Certificate-Bound Tokens
|
||||
|
||||
**Configuration model**
|
||||
- Reuse `SenderConstraints.Mtls` described above; include:
|
||||
- `enforceForAudiences` list (defaults `signer`, `attestor`, `scheduler`).
|
||||
- `certificateRotationGraceSeconds` for overlap.
|
||||
- `allowedClientCertificateAuthorities` absolute paths.
|
||||
|
||||
**Kestrel/TLS pipeline**
|
||||
- Configure Kestrel with `ClientCertificateMode.AllowCertificate` globally and implement middleware that enforces certificate presence only when the resolved client requires mTLS.
|
||||
- Add `IAuthorityClientCertificateValidator` that validates presented certificate chain, SANs (`dns`, `uri`, optional SPIFFE), and thumbprint matches one of the stored bindings.
|
||||
- Cache validation results per connection id to avoid rehashing on every request.
|
||||
|
||||
**Client registration & storage**
|
||||
- Extend `AuthorityClientDocument` with `List<AuthorityClientCertificateBinding>` containing:
|
||||
- `Thumbprint`, `SerialNumber`, `Subject`, `NotBefore`, `NotAfter`, `Sans`, `CreatedAt`, `UpdatedAt`, `Label`.
|
||||
- Provide admin API mutations (`/admin/clients/{id}/certificates`) for ops tooling (deferred implementation but schema ready).
|
||||
- Update plugin provisioning store (`StandardClientProvisioningStore`) to map descriptors with certificate bindings and `senderConstraint`.
|
||||
- Persist binding state in Mongo migrations (index on `{clientId, thumbprint}`).
|
||||
|
||||
**Token issuance & introspection**
|
||||
- Add a transaction property capturing the validated client certificate thumbprint.
|
||||
- `HandleClientCredentialsHandler`:
|
||||
- When mTLS required, ensure certificate info present; reject otherwise.
|
||||
- Stamp `cnf` claim: `principal.SetClaim("cnf", JsonSerializer.Serialize(new { x5t#S256 = thumbprint }))`.
|
||||
- Store binding metadata in issued token document for audit.
|
||||
- Update `ValidateAccessTokenHandler` and introspection responses to surface `cnf.x5t#S256`.
|
||||
- Ensure refresh tokens (if ever enabled) copy the binding data.
|
||||
|
||||
**Auditing & observability**
|
||||
- Audit events:
|
||||
- `authority.mtls.handshake` (success/failure, clientId, thumbprint, issuer, subject).
|
||||
- `authority.mtls.binding.missing` when a required client posts without a cert.
|
||||
- Metrics:
|
||||
- `authority_mtls_handshakes_total{result}`.
|
||||
- `authority_mtls_certificate_rotations_total`.
|
||||
- Logs include `authority.sender_constraint=mtls`, `authority.mtls_thumbprint`, `authority.mtls_subject`.
|
||||
|
||||
**Testing**
|
||||
- Unit tests for certificate validation rules (SAN mismatches, expiry, CA trust).
|
||||
- Integration tests running Kestrel with test certificates:
|
||||
- Successful token issuance with bound certificate.
|
||||
- Request without certificate → `invalid_client`.
|
||||
- Token introspection reveals `cnf.x5t#S256`.
|
||||
- Rotation scenario (old + new cert allowed during grace window).
|
||||
|
||||
## Implementation Checklist
|
||||
|
||||
**DPoP work-stream**
|
||||
1. Extract shared validator into `StellaOps.Auth.Security`; update Scanner references.
|
||||
2. Introduce configuration classes and bind from YAML/environment.
|
||||
3. Implement nonce store (Redis + in-memory), handler integration, and OpenIddict transaction plumbing.
|
||||
4. Stamp `cnf.jkt`, audit events, and metrics; update Mongo documents and migrations.
|
||||
5. Extend docs: `docs/ARCHITECTURE_AUTHORITY.md`, `docs/security/audit-events.md`, `docs/security/rate-limits.md`, CLI/UI references.
|
||||
|
||||
**mTLS work-stream**
|
||||
1. Extend client document/schema and provisioning stores with certificate bindings + sender constraint flag.
|
||||
2. Configure Kestrel/middleware for optional client certificates and validation service.
|
||||
3. Update token issuance/introspection to honour certificate bindings and emit `cnf.x5t#S256`.
|
||||
4. Add auditing/metrics and integration tests (happy path + failure).
|
||||
5. Refresh operator documentation (`docs/ops/authority-backup-restore.md`, `docs/ops/authority-monitoring.md`, sample `authority.yaml`) to cover certificate lifecycle.
|
||||
|
||||
Both streams should conclude with `dotnet test src/StellaOps.Authority.sln` and documentation cross-links so dependent guilds can unblock UI/Signer work.
|
||||
52
docs/dev/authority-plugin-di-coordination.md
Normal file
52
docs/dev/authority-plugin-di-coordination.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Authority Plug-in Scoped Service Coordination
|
||||
|
||||
> Created: 2025-10-19 — Plugin Platform Guild & Authority Core
|
||||
> Status: Scheduled (session confirmed for 2025-10-20 15:00–16:00 UTC)
|
||||
|
||||
This document tracks preparation, agenda, and outcomes for the scoped-service workshop required before implementing PLUGIN-DI-08-002.
|
||||
|
||||
## Objectives
|
||||
|
||||
- Inventory Authority plug-in surfaces that need scoped service lifetimes.
|
||||
- Confirm session/scope handling for identity-provider registrars and background jobs.
|
||||
- Assign follow-up tasks/actions with owners and due dates.
|
||||
|
||||
## Scheduling Snapshot
|
||||
|
||||
- **Meeting time:** 2025-10-20 15:00–16:00 UTC (10:00–11:00 CDT / 08:00–09:00 PDT).
|
||||
- **Facilitator:** Plugin Platform Guild — Alicia Rivera.
|
||||
- **Attendees (confirmed):** Authority Core — Jasmin Patel; Authority Security Guild — Mohan Singh; Plugin Platform — Alicia Rivera, Leah Chen.
|
||||
- **Optional invitees:** DevOps liaison — Sofia Ortega (accepted).
|
||||
- **Logistics:** Invites sent via shared calendar on 2025-10-19 15:30 UTC with Teams bridge + offline dial-in. Meeting notes will be captured here.
|
||||
- **Preparation deadline:** 2025-10-20 12:00 UTC — complete checklist below.
|
||||
|
||||
## Pre-work Checklist
|
||||
|
||||
- Review `ServiceBindingAttribute` contract introduced by PLUGIN-DI-08-001.
|
||||
- Collect existing Authority plug-in registration code paths to evaluate.
|
||||
- Audit background jobs that assume singleton lifetimes.
|
||||
- Identify plug-in health checks/telemetry surfaces impacted by scoped lifetimes.
|
||||
|
||||
### Pre-work References
|
||||
|
||||
- _Add links, file paths, or notes here prior to the session._
|
||||
|
||||
## Draft Agenda
|
||||
|
||||
1. Context recap (5 min) — why scoped DI is needed; summary of PLUGIN-DI-08-001 changes.
|
||||
2. Authority plug-in surfaces (15 min) — registrars, background services, telemetry.
|
||||
3. Session handling strategy (10 min) — scope creation semantics, cancellation propagation.
|
||||
4. Action items & owners (10 min) — capture code/docs/test tasks with due dates.
|
||||
5. Risks & follow-ups (5 min) — dependencies, rollout sequencing.
|
||||
|
||||
## Notes
|
||||
|
||||
- _Pending coordination session; populate afterwards._
|
||||
|
||||
## Action Item Log
|
||||
|
||||
| Item | Owner | Due | Status | Notes |
|
||||
|------|-------|-----|--------|-------|
|
||||
| Confirm meeting time | Alicia Rivera | 2025-10-19 15:30 UTC | DONE | Calendar invite sent; all required attendees accepted |
|
||||
| Compile Authority plug-in DI entry points | Jasmin Patel | 2025-10-20 | IN PROGRESS | Gather current Authority plug-in registrars, background jobs, and helper factories that assume singleton lifetimes; add the list with file paths to **Pre-work References** in this document before 2025-10-20 12:00 UTC. |
|
||||
| Outline scoped-session pattern for background jobs | Leah Chen | Post-session | BLOCKED | Requires meeting outcomes |
|
||||
@@ -1,45 +1,45 @@
|
||||
# Feedser Fixture Maintenance
|
||||
# Concelier Fixture Maintenance
|
||||
|
||||
Feedser uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the
|
||||
Concelier uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the
|
||||
fixture sets, where they live, and how to regenerate them safely.
|
||||
|
||||
---
|
||||
|
||||
## GHSA ↔ OSV parity fixtures
|
||||
|
||||
- **Location:** `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.*.json`
|
||||
- **Location:** `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.*.json`
|
||||
- **Purpose:** Exercised by `OsvGhsaParityRegressionTests` to ensure OSV + GHSA outputs stay aligned on aliases,
|
||||
ranges, references, and credits.
|
||||
- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`)
|
||||
- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`)
|
||||
or execute the fixture updater (`dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj`). Both paths
|
||||
normalise timestamps and canonical ordering.
|
||||
- **SemVer provenance:** The regenerated fixtures should show `normalizedVersions[].notes` in the
|
||||
`osv:{ecosystem}:{advisoryId}:{identifier}` shape emitted by `SemVerRangeRuleBuilder`. Confirm the
|
||||
constraints and notes line up with GHSA/NVD composites before committing.
|
||||
- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj` to confirm parity.
|
||||
- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj` to confirm parity.
|
||||
|
||||
## GHSA credit parity fixtures
|
||||
|
||||
- **Location:** `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json`
|
||||
- **Location:** `src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json`
|
||||
- **Purpose:** Exercised by `GhsaCreditParityRegressionTests` to guarantee GHSA/NVD/OSV acknowledgements remain in lockstep.
|
||||
- **Regeneration:** `dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj` rewrites all three canonical snapshots.
|
||||
- **Verification:** `dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj`.
|
||||
- **Verification:** `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj`.
|
||||
|
||||
> Always commit fixture changes together with the code that motivated them and reference the regression test that guards the behaviour.
|
||||
|
||||
## Apple security update fixtures
|
||||
|
||||
- **Location:** `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`.
|
||||
- **Location:** `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`.
|
||||
- **Purpose:** Exercised by `AppleLiveRegressionTests` to guarantee the Apple HTML parser and mapper stay deterministic while covering Rapid Security Responses and multi-device advisories.
|
||||
- **Regeneration:** Use the helper scripts (`scripts/update-apple-fixtures.sh` or `scripts/update-apple-fixtures.ps1`). They export `UPDATE_APPLE_FIXTURES=1`, propagate the flag through `WSLENV`, touch `.update-apple-fixtures`, and then run the Apple test project. This keeps WSL/VSCode test invocations in sync while the refresh workflow fetches live Apple support pages, sanitises them, and rewrites both the HTML and expected DTO snapshots with normalised ordering.
|
||||
- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj` without the env var to confirm determinism.
|
||||
- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the env var to confirm determinism.
|
||||
|
||||
> **Tip for other connector owners:** mirror the sentinel + `WSLENV` pattern (`touch .update-<connector>-fixtures`, append the env var via `WSLENV`) when you add fixture refresh scripts so contributors running under WSL inherit the regeneration flag automatically.
|
||||
|
||||
## KISA advisory fixtures
|
||||
|
||||
- **Location:** `src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-{feed,detail}.(xml|json)`
|
||||
- **Location:** `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-{feed,detail}.(xml|json)`
|
||||
- **Purpose:** Used by `KisaConnectorTests` to verify Hangul-aware fetch → parse → map flows and to assert telemetry counters stay wired.
|
||||
- **Regeneration:** `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj`
|
||||
- **Regeneration:** `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`
|
||||
- **Verification:** Re-run the same test suite without the env var; confirm advisory content remains NFC-normalised and HTML is sanitised. Metrics assertions will fail if counters drift.
|
||||
- **Localisation note:** RSS `category` values (e.g. `취약점정보`) remain in Hangul—do not translate them in fixtures; they feed directly into metrics/log tags.
|
||||
|
||||
@@ -4,7 +4,7 @@ The KISA/KNVD connector now ships with structured telemetry, richer logging, and
|
||||
|
||||
## Telemetry counters
|
||||
|
||||
All metrics are emitted from `KisaDiagnostics` (`Meter` name `StellaOps.Feedser.Source.Kisa`).
|
||||
All metrics are emitted from `KisaDiagnostics` (`Meter` name `StellaOps.Concelier.Connector.Kisa`).
|
||||
|
||||
| Metric | Description | Tags |
|
||||
| --- | --- | --- |
|
||||
@@ -39,7 +39,7 @@ The messages use structured properties (`Idx`, `Category`, `DocumentId`, `Severi
|
||||
- Hangul fields (`title`, `summary`, `category`, `reference.label`, product vendor/name) are normalised to NFC before storage. Sample category `취약점정보` roughly translates to “vulnerability information”.
|
||||
- Advisory HTML is sanitised via `HtmlContentSanitizer`, stripping script/style while preserving inline anchors for translation pipelines.
|
||||
- Metrics carry Hangul `category` tags and logging keeps Hangul strings intact; this ensures air-gapped operators can validate native-language content without relying on MT.
|
||||
- Fixtures live under `src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/`. Regenerate with `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj`.
|
||||
- Fixtures live under `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/`. Regenerate with `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`.
|
||||
- The regression suite asserts canonical mapping, state cleanup, and telemetry counters (`KisaConnectorTests.Telemetry_RecordsMetrics`) so QA can track instrumentation drift.
|
||||
|
||||
For operator docs, link to this brief when documenting Hangul handling or counter dashboards so localisation reviewers have a single reference point.
|
||||
|
||||
@@ -1,154 +1,154 @@
|
||||
# Feedser SemVer Merge Playbook (Sprint 1–2)
|
||||
|
||||
This playbook describes how the merge layer and connector teams should emit the new SemVer primitives introduced in Sprint 1–2, how those primitives become normalized version rules, and how downstream jobs query them deterministically.
|
||||
|
||||
## 1. What landed in Sprint 1–2
|
||||
|
||||
- `RangePrimitives.SemVer` now infers a canonical `style` (`range`, `exact`, `lt`, `lte`, `gt`, `gte`) and captures `exactValue` when the constraint is a single version.
|
||||
- `NormalizedVersionRule` documents the analytics-friendly projection of each `AffectedPackage` coverage entry and is persisted alongside legacy `versionRanges`.
|
||||
- `AdvisoryProvenance.decisionReason` records whether merge resolution favored precedence, freshness, or a tie-breaker comparison.
|
||||
|
||||
See `src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions.
|
||||
|
||||
## 2. Mapper pattern
|
||||
|
||||
Connectors should emit SemVer primitives as soon as they can normalize a vendor constraint. The helper `SemVerPrimitiveExtensions.ToNormalizedVersionRule` turns those primitives into the persisted rules:
|
||||
|
||||
```csharp
|
||||
var primitive = new SemVerPrimitive(
|
||||
introduced: "1.2.3",
|
||||
introducedInclusive: true,
|
||||
fixed: "2.0.0",
|
||||
fixedInclusive: false,
|
||||
lastAffected: null,
|
||||
lastAffectedInclusive: false,
|
||||
constraintExpression: ">=1.2.3 <2.0.0",
|
||||
exactValue: null);
|
||||
|
||||
var rule = primitive.ToNormalizedVersionRule(notes: "nvd:CVE-2025-1234");
|
||||
// rule => scheme=semver, type=range, min=1.2.3, minInclusive=true, max=2.0.0, maxInclusive=false
|
||||
```
|
||||
|
||||
If you omit the optional `notes` argument, `ToNormalizedVersionRule` now falls back to the primitive’s `ConstraintExpression`, ensuring the original comparator expression is preserved for provenance/audit queries.
|
||||
|
||||
Emit the resulting rule inside `AffectedPackage.NormalizedVersions` while continuing to populate `AffectedVersionRange.RangeExpression` for backward compatibility.
|
||||
|
||||
## 3. Merge dedupe flow
|
||||
|
||||
During merge, feed all package candidates through `NormalizedVersionRuleComparer.Instance` prior to persistence. The comparer orders by scheme → type → min → minInclusive → max → maxInclusive → value → notes, guaranteeing consistent document layout and making `$unwind` pipelines deterministic.
|
||||
|
||||
If multiple connectors emit identical constraints, the merge layer should:
|
||||
|
||||
1. Combine provenance entries (preserving one per source).
|
||||
2. Preserve a single normalized rule instance (thanks to `NormalizedVersionRuleEqualityComparer.Instance`).
|
||||
3. Attach `decisionReason="precedence"` if one source overrides another.
|
||||
|
||||
## 4. Example Mongo pipeline
|
||||
|
||||
Use the following aggregation to locate advisories that affect a specific SemVer:
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $match: { "affectedPackages.type": "semver", "affectedPackages.identifier": "pkg:npm/lodash" } },
|
||||
{ $unwind: "$affectedPackages" },
|
||||
{ $unwind: "$affectedPackages.normalizedVersions" },
|
||||
{ $match: {
|
||||
$or: [
|
||||
{ "affectedPackages.normalizedVersions.type": "exact",
|
||||
"affectedPackages.normalizedVersions.value": "4.17.21" },
|
||||
{ "affectedPackages.normalizedVersions.type": "range",
|
||||
"affectedPackages.normalizedVersions.min": { $lte: "4.17.21" },
|
||||
"affectedPackages.normalizedVersions.max": { $gt: "4.17.21" } },
|
||||
{ "affectedPackages.normalizedVersions.type": "gte",
|
||||
"affectedPackages.normalizedVersions.min": { $lte: "4.17.21" } },
|
||||
{ "affectedPackages.normalizedVersions.type": "lte",
|
||||
"affectedPackages.normalizedVersions.max": { $gte: "4.17.21" } }
|
||||
]
|
||||
}},
|
||||
{ $project: { advisoryKey: 1, title: 1, "affectedPackages.identifier": 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Pair this query with the indexes listed in [Normalized Versions Query Guide](mongo_indices.md).
|
||||
|
||||
## 5. Recommended indexes
|
||||
|
||||
| Collection | Index | Purpose |
|
||||
|------------|-------|---------|
|
||||
| `advisory` | `{ "affectedPackages.identifier": 1, "affectedPackages.normalizedVersions.scheme": 1, "affectedPackages.normalizedVersions.type": 1 }` (compound, multikey) | Speeds up `$match` on identifier + rule style. |
|
||||
| `advisory` | `{ "affectedPackages.normalizedVersions.value": 1 }` (sparse) | Optimizes lookups for exact version hits. |
|
||||
|
||||
Coordinate with the Storage team when enabling these indexes so deployment windows account for collection size.
|
||||
|
||||
## 6. Dual-write rollout
|
||||
|
||||
Follow the operational checklist in `docs/ops/migrations/SEMVER_STYLE.md`. The summary:
|
||||
|
||||
1. **Dual write (now)** – emit both legacy `versionRanges` and the new `normalizedVersions`.
|
||||
2. **Backfill** – follow the storage migration in `docs/ops/migrations/SEMVER_STYLE.md` to rewrite historical advisories before switching consumers.
|
||||
3. **Verify** – run the aggregation above (with `explain("executionStats")`) to ensure the new indexes are used.
|
||||
4. **Cutover** – after consumers switch to normalized rules, mark the old `rangeExpression` as deprecated.
|
||||
|
||||
## 7. Checklist for connectors & merge
|
||||
|
||||
- [ ] Populate `SemVerPrimitive` for every SemVer-friendly constraint.
|
||||
- [ ] Call `ToNormalizedVersionRule` and store the result.
|
||||
- [ ] Emit provenance masks covering both `versionRanges[].primitives.semver` and `normalizedVersions[]`.
|
||||
- [ ] Ensure merge deduping relies on the canonical comparer.
|
||||
- [ ] Capture merge decisions via `decisionReason`.
|
||||
- [ ] Confirm integration tests include fixtures with normalized rules and SemVer styles.
|
||||
|
||||
For deeper query examples and maintenance tasks, continue with [Normalized Versions Query Guide](mongo_indices.md).
|
||||
|
||||
## 8. Storage projection reference
|
||||
|
||||
`NormalizedVersionDocumentFactory` copies each normalized rule into MongoDB using the shape below. Use this as a contract when reviewing connector fixtures or diagnosing merge/storage diffs:
|
||||
|
||||
```json
|
||||
{
|
||||
"packageId": "pkg:npm/example",
|
||||
"packageType": "npm",
|
||||
"scheme": "semver",
|
||||
"type": "range",
|
||||
"style": "range",
|
||||
"min": "1.2.3",
|
||||
"minInclusive": true,
|
||||
"max": "2.0.0",
|
||||
"maxInclusive": false,
|
||||
"value": null,
|
||||
"notes": "ghsa:GHSA-xxxx-yyyy",
|
||||
"decisionReason": "ghsa-precedence-over-nvd",
|
||||
"constraint": ">= 1.2.3 < 2.0.0",
|
||||
"source": "ghsa",
|
||||
"recordedAt": "2025-10-11T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
For distro-specific ranges (`nevra`, `evr`) the same envelope applies with `scheme` switched accordingly. Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"packageId": "bash",
|
||||
"packageType": "rpm",
|
||||
"scheme": "nevra",
|
||||
"type": "range",
|
||||
"style": "range",
|
||||
"min": "0:4.4.18-2.el7",
|
||||
"minInclusive": true,
|
||||
"max": "0:4.4.20-1.el7",
|
||||
"maxInclusive": false,
|
||||
"value": null,
|
||||
"notes": "redhat:RHSA-2025:1234",
|
||||
"decisionReason": "rhel-priority-over-nvd",
|
||||
"constraint": "<= 0:4.4.20-1.el7",
|
||||
"source": "redhat",
|
||||
"recordedAt": "2025-10-11T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
If a new scheme is required (for example, `apple.build` or `ios.semver`), raise it with the Models team before emitting documents so merge comparers and hashing logic can incorporate the change deterministically.
|
||||
|
||||
## 9. Observability signals
|
||||
|
||||
- `feedser.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) – increments once per normalized rule retained after precedence merge.
|
||||
- `feedser.merge.normalized_rules_missing` (counter, tags: `package_type`) – increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet.
|
||||
# Concelier SemVer Merge Playbook (Sprint 1–2)
|
||||
|
||||
This playbook describes how the merge layer and connector teams should emit the new SemVer primitives introduced in Sprint 1–2, how those primitives become normalized version rules, and how downstream jobs query them deterministically.
|
||||
|
||||
## 1. What landed in Sprint 1–2
|
||||
|
||||
- `RangePrimitives.SemVer` now infers a canonical `style` (`range`, `exact`, `lt`, `lte`, `gt`, `gte`) and captures `exactValue` when the constraint is a single version.
|
||||
- `NormalizedVersionRule` documents the analytics-friendly projection of each `AffectedPackage` coverage entry and is persisted alongside legacy `versionRanges`.
|
||||
- `AdvisoryProvenance.decisionReason` records whether merge resolution favored precedence, freshness, or a tie-breaker comparison.
|
||||
|
||||
See `src/StellaOps.Concelier.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions.
|
||||
|
||||
## 2. Mapper pattern
|
||||
|
||||
Connectors should emit SemVer primitives as soon as they can normalize a vendor constraint. The helper `SemVerPrimitiveExtensions.ToNormalizedVersionRule` turns those primitives into the persisted rules:
|
||||
|
||||
```csharp
|
||||
var primitive = new SemVerPrimitive(
|
||||
introduced: "1.2.3",
|
||||
introducedInclusive: true,
|
||||
fixed: "2.0.0",
|
||||
fixedInclusive: false,
|
||||
lastAffected: null,
|
||||
lastAffectedInclusive: false,
|
||||
constraintExpression: ">=1.2.3 <2.0.0",
|
||||
exactValue: null);
|
||||
|
||||
var rule = primitive.ToNormalizedVersionRule(notes: "nvd:CVE-2025-1234");
|
||||
// rule => scheme=semver, type=range, min=1.2.3, minInclusive=true, max=2.0.0, maxInclusive=false
|
||||
```
|
||||
|
||||
If you omit the optional `notes` argument, `ToNormalizedVersionRule` now falls back to the primitive’s `ConstraintExpression`, ensuring the original comparator expression is preserved for provenance/audit queries.
|
||||
|
||||
Emit the resulting rule inside `AffectedPackage.NormalizedVersions` while continuing to populate `AffectedVersionRange.RangeExpression` for backward compatibility.
|
||||
|
||||
## 3. Merge dedupe flow
|
||||
|
||||
During merge, feed all package candidates through `NormalizedVersionRuleComparer.Instance` prior to persistence. The comparer orders by scheme → type → min → minInclusive → max → maxInclusive → value → notes, guaranteeing consistent document layout and making `$unwind` pipelines deterministic.
|
||||
|
||||
If multiple connectors emit identical constraints, the merge layer should:
|
||||
|
||||
1. Combine provenance entries (preserving one per source).
|
||||
2. Preserve a single normalized rule instance (thanks to `NormalizedVersionRuleEqualityComparer.Instance`).
|
||||
3. Attach `decisionReason="precedence"` if one source overrides another.
|
||||
|
||||
## 4. Example Mongo pipeline
|
||||
|
||||
Use the following aggregation to locate advisories that affect a specific SemVer:
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $match: { "affectedPackages.type": "semver", "affectedPackages.identifier": "pkg:npm/lodash" } },
|
||||
{ $unwind: "$affectedPackages" },
|
||||
{ $unwind: "$affectedPackages.normalizedVersions" },
|
||||
{ $match: {
|
||||
$or: [
|
||||
{ "affectedPackages.normalizedVersions.type": "exact",
|
||||
"affectedPackages.normalizedVersions.value": "4.17.21" },
|
||||
{ "affectedPackages.normalizedVersions.type": "range",
|
||||
"affectedPackages.normalizedVersions.min": { $lte: "4.17.21" },
|
||||
"affectedPackages.normalizedVersions.max": { $gt: "4.17.21" } },
|
||||
{ "affectedPackages.normalizedVersions.type": "gte",
|
||||
"affectedPackages.normalizedVersions.min": { $lte: "4.17.21" } },
|
||||
{ "affectedPackages.normalizedVersions.type": "lte",
|
||||
"affectedPackages.normalizedVersions.max": { $gte: "4.17.21" } }
|
||||
]
|
||||
}},
|
||||
{ $project: { advisoryKey: 1, title: 1, "affectedPackages.identifier": 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Pair this query with the indexes listed in [Normalized Versions Query Guide](mongo_indices.md).
|
||||
|
||||
## 5. Recommended indexes
|
||||
|
||||
| Collection | Index | Purpose |
|
||||
|------------|-------|---------|
|
||||
| `advisory` | `{ "affectedPackages.identifier": 1, "affectedPackages.normalizedVersions.scheme": 1, "affectedPackages.normalizedVersions.type": 1 }` (compound, multikey) | Speeds up `$match` on identifier + rule style. |
|
||||
| `advisory` | `{ "affectedPackages.normalizedVersions.value": 1 }` (sparse) | Optimizes lookups for exact version hits. |
|
||||
|
||||
Coordinate with the Storage team when enabling these indexes so deployment windows account for collection size.
|
||||
|
||||
## 6. Dual-write rollout
|
||||
|
||||
Follow the operational checklist in `docs/ops/migrations/SEMVER_STYLE.md`. The summary:
|
||||
|
||||
1. **Dual write (now)** – emit both legacy `versionRanges` and the new `normalizedVersions`.
|
||||
2. **Backfill** – follow the storage migration in `docs/ops/migrations/SEMVER_STYLE.md` to rewrite historical advisories before switching consumers.
|
||||
3. **Verify** – run the aggregation above (with `explain("executionStats")`) to ensure the new indexes are used.
|
||||
4. **Cutover** – after consumers switch to normalized rules, mark the old `rangeExpression` as deprecated.
|
||||
|
||||
## 7. Checklist for connectors & merge
|
||||
|
||||
- [ ] Populate `SemVerPrimitive` for every SemVer-friendly constraint.
|
||||
- [ ] Call `ToNormalizedVersionRule` and store the result.
|
||||
- [ ] Emit provenance masks covering both `versionRanges[].primitives.semver` and `normalizedVersions[]`.
|
||||
- [ ] Ensure merge deduping relies on the canonical comparer.
|
||||
- [ ] Capture merge decisions via `decisionReason`.
|
||||
- [ ] Confirm integration tests include fixtures with normalized rules and SemVer styles.
|
||||
|
||||
For deeper query examples and maintenance tasks, continue with [Normalized Versions Query Guide](mongo_indices.md).
|
||||
|
||||
## 8. Storage projection reference
|
||||
|
||||
`NormalizedVersionDocumentFactory` copies each normalized rule into MongoDB using the shape below. Use this as a contract when reviewing connector fixtures or diagnosing merge/storage diffs:
|
||||
|
||||
```json
|
||||
{
|
||||
"packageId": "pkg:npm/example",
|
||||
"packageType": "npm",
|
||||
"scheme": "semver",
|
||||
"type": "range",
|
||||
"style": "range",
|
||||
"min": "1.2.3",
|
||||
"minInclusive": true,
|
||||
"max": "2.0.0",
|
||||
"maxInclusive": false,
|
||||
"value": null,
|
||||
"notes": "ghsa:GHSA-xxxx-yyyy",
|
||||
"decisionReason": "ghsa-precedence-over-nvd",
|
||||
"constraint": ">= 1.2.3 < 2.0.0",
|
||||
"source": "ghsa",
|
||||
"recordedAt": "2025-10-11T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
For distro-specific ranges (`nevra`, `evr`) the same envelope applies with `scheme` switched accordingly. Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"packageId": "bash",
|
||||
"packageType": "rpm",
|
||||
"scheme": "nevra",
|
||||
"type": "range",
|
||||
"style": "range",
|
||||
"min": "0:4.4.18-2.el7",
|
||||
"minInclusive": true,
|
||||
"max": "0:4.4.20-1.el7",
|
||||
"maxInclusive": false,
|
||||
"value": null,
|
||||
"notes": "redhat:RHSA-2025:1234",
|
||||
"decisionReason": "rhel-priority-over-nvd",
|
||||
"constraint": "<= 0:4.4.20-1.el7",
|
||||
"source": "redhat",
|
||||
"recordedAt": "2025-10-11T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
If a new scheme is required (for example, `apple.build` or `ios.semver`), raise it with the Models team before emitting documents so merge comparers and hashing logic can incorporate the change deterministically.
|
||||
|
||||
## 9. Observability signals
|
||||
|
||||
- `concelier.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) – increments once per normalized rule retained after precedence merge.
|
||||
- `concelier.merge.normalized_rules_missing` (counter, tags: `package_type`) – increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet.
|
||||
|
||||
@@ -1,108 +1,108 @@
|
||||
# Normalized Versions Query Guide
|
||||
|
||||
This guide complements the Sprint 1–2 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`.
|
||||
|
||||
For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section 8 of the [Feedser SemVer Merge Playbook](merge_semver_playbook.md).
|
||||
|
||||
## 1. Recommended indexes
|
||||
|
||||
When `feedser.storage.enableSemVerStyle` is enabled, advisories expose a flattened
|
||||
`normalizedVersions` array at the document root. Create these indexes in `mongosh`
|
||||
after the migration completes (adjust collection name if you use a prefix):
|
||||
|
||||
```javascript
|
||||
db.advisories.createIndex(
|
||||
{
|
||||
"normalizedVersions.packageId": 1,
|
||||
"normalizedVersions.scheme": 1,
|
||||
"normalizedVersions.type": 1
|
||||
},
|
||||
{ name: "advisory_normalizedVersions_pkg_scheme_type" }
|
||||
);
|
||||
|
||||
db.advisories.createIndex(
|
||||
{ "normalizedVersions.value": 1 },
|
||||
{ name: "advisory_normalizedVersions_value", sparse: true }
|
||||
);
|
||||
```
|
||||
|
||||
- The compound index accelerates `$match` stages that filter by package identifier and rule style without unwinding `affectedPackages`.
|
||||
- The sparse index keeps storage costs low while supporting pure exact-version lookups (type `exact`).
|
||||
|
||||
The storage bootstrapper creates the same indexes automatically when the feature flag is enabled.
|
||||
|
||||
## 2. Query patterns
|
||||
|
||||
### 2.1 Determine if a specific version is affected
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $match: { "normalizedVersions.packageId": "pkg:npm/lodash" } },
|
||||
{ $unwind: "$normalizedVersions" },
|
||||
{ $match: {
|
||||
$or: [
|
||||
{ "normalizedVersions.type": "exact",
|
||||
"normalizedVersions.value": "4.17.21" },
|
||||
{ "normalizedVersions.type": "range",
|
||||
"normalizedVersions.min": { $lte: "4.17.21" },
|
||||
"normalizedVersions.max": { $gt: "4.17.21" } },
|
||||
{ "normalizedVersions.type": "gte",
|
||||
"normalizedVersions.min": { $lte: "4.17.21" } },
|
||||
{ "normalizedVersions.type": "lte",
|
||||
"normalizedVersions.max": { $gte: "4.17.21" } }
|
||||
]
|
||||
}},
|
||||
{ $project: { advisoryKey: 1, title: 1, "normalizedVersions.packageId": 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Use this pipeline during Sprint 2 staging validation runs. Invoke `explain("executionStats")` to confirm the compound index is selected.
|
||||
|
||||
### 2.2 Locate advisories missing normalized rules
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $match: { $or: [
|
||||
{ "normalizedVersions": { $exists: false } },
|
||||
{ "normalizedVersions": { $size: 0 } }
|
||||
] } },
|
||||
{ $project: { advisoryKey: 1, affectedPackages: 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Run this query after backfill jobs to identify gaps that still rely solely on `rangeExpression`.
|
||||
|
||||
### 2.3 Deduplicate overlapping rules
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $unwind: "$normalizedVersions" },
|
||||
{ $group: {
|
||||
_id: {
|
||||
identifier: "$normalizedVersions.packageId",
|
||||
scheme: "$normalizedVersions.scheme",
|
||||
type: "$normalizedVersions.type",
|
||||
min: "$normalizedVersions.min",
|
||||
minInclusive: "$normalizedVersions.minInclusive",
|
||||
max: "$normalizedVersions.max",
|
||||
maxInclusive: "$normalizedVersions.maxInclusive",
|
||||
value: "$normalizedVersions.value"
|
||||
},
|
||||
advisories: { $addToSet: "$advisoryKey" },
|
||||
notes: { $addToSet: "$normalizedVersions.notes" }
|
||||
}},
|
||||
{ $match: { "advisories.1": { $exists: true } } },
|
||||
{ $sort: { "_id.identifier": 1, "_id.type": 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Use this to confirm the merge dedupe logic keeps only one normalized rule per unique constraint.
|
||||
|
||||
## 3. Operational checklist
|
||||
|
||||
- [ ] Create the indexes in staging before toggling dual-write in production.
|
||||
- [ ] Capture explain plans and attach them to the release notes.
|
||||
- [ ] Notify downstream services that consume advisory snapshots about the new `normalizedVersions` array.
|
||||
- [ ] Update export fixtures once dedupe verification passes.
|
||||
|
||||
Additional background and mapper examples live in [Feedser SemVer Merge Playbook](merge_semver_playbook.md).
|
||||
# Normalized Versions Query Guide
|
||||
|
||||
This guide complements the Sprint 1–2 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`.
|
||||
|
||||
For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section 8 of the [Concelier SemVer Merge Playbook](merge_semver_playbook.md).
|
||||
|
||||
## 1. Recommended indexes
|
||||
|
||||
When `concelier.storage.enableSemVerStyle` is enabled, advisories expose a flattened
|
||||
`normalizedVersions` array at the document root. Create these indexes in `mongosh`
|
||||
after the migration completes (adjust collection name if you use a prefix):
|
||||
|
||||
```javascript
|
||||
db.advisories.createIndex(
|
||||
{
|
||||
"normalizedVersions.packageId": 1,
|
||||
"normalizedVersions.scheme": 1,
|
||||
"normalizedVersions.type": 1
|
||||
},
|
||||
{ name: "advisory_normalizedVersions_pkg_scheme_type" }
|
||||
);
|
||||
|
||||
db.advisories.createIndex(
|
||||
{ "normalizedVersions.value": 1 },
|
||||
{ name: "advisory_normalizedVersions_value", sparse: true }
|
||||
);
|
||||
```
|
||||
|
||||
- The compound index accelerates `$match` stages that filter by package identifier and rule style without unwinding `affectedPackages`.
|
||||
- The sparse index keeps storage costs low while supporting pure exact-version lookups (type `exact`).
|
||||
|
||||
The storage bootstrapper creates the same indexes automatically when the feature flag is enabled.
|
||||
|
||||
## 2. Query patterns
|
||||
|
||||
### 2.1 Determine if a specific version is affected
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $match: { "normalizedVersions.packageId": "pkg:npm/lodash" } },
|
||||
{ $unwind: "$normalizedVersions" },
|
||||
{ $match: {
|
||||
$or: [
|
||||
{ "normalizedVersions.type": "exact",
|
||||
"normalizedVersions.value": "4.17.21" },
|
||||
{ "normalizedVersions.type": "range",
|
||||
"normalizedVersions.min": { $lte: "4.17.21" },
|
||||
"normalizedVersions.max": { $gt: "4.17.21" } },
|
||||
{ "normalizedVersions.type": "gte",
|
||||
"normalizedVersions.min": { $lte: "4.17.21" } },
|
||||
{ "normalizedVersions.type": "lte",
|
||||
"normalizedVersions.max": { $gte: "4.17.21" } }
|
||||
]
|
||||
}},
|
||||
{ $project: { advisoryKey: 1, title: 1, "normalizedVersions.packageId": 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Use this pipeline during Sprint 2 staging validation runs. Invoke `explain("executionStats")` to confirm the compound index is selected.
|
||||
|
||||
### 2.2 Locate advisories missing normalized rules
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $match: { $or: [
|
||||
{ "normalizedVersions": { $exists: false } },
|
||||
{ "normalizedVersions": { $size: 0 } }
|
||||
] } },
|
||||
{ $project: { advisoryKey: 1, affectedPackages: 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Run this query after backfill jobs to identify gaps that still rely solely on `rangeExpression`.
|
||||
|
||||
### 2.3 Deduplicate overlapping rules
|
||||
|
||||
```javascript
|
||||
db.advisories.aggregate([
|
||||
{ $unwind: "$normalizedVersions" },
|
||||
{ $group: {
|
||||
_id: {
|
||||
identifier: "$normalizedVersions.packageId",
|
||||
scheme: "$normalizedVersions.scheme",
|
||||
type: "$normalizedVersions.type",
|
||||
min: "$normalizedVersions.min",
|
||||
minInclusive: "$normalizedVersions.minInclusive",
|
||||
max: "$normalizedVersions.max",
|
||||
maxInclusive: "$normalizedVersions.maxInclusive",
|
||||
value: "$normalizedVersions.value"
|
||||
},
|
||||
advisories: { $addToSet: "$advisoryKey" },
|
||||
notes: { $addToSet: "$normalizedVersions.notes" }
|
||||
}},
|
||||
{ $match: { "advisories.1": { $exists: true } } },
|
||||
{ $sort: { "_id.identifier": 1, "_id.type": 1 } }
|
||||
]);
|
||||
```
|
||||
|
||||
Use this to confirm the merge dedupe logic keeps only one normalized rule per unique constraint.
|
||||
|
||||
## 3. Operational checklist
|
||||
|
||||
- [ ] Create the indexes in staging before toggling dual-write in production.
|
||||
- [ ] Capture explain plans and attach them to the release notes.
|
||||
- [ ] Notify downstream services that consume advisory snapshots about the new `normalizedVersions` array.
|
||||
- [ ] Update export fixtures once dedupe verification passes.
|
||||
|
||||
Additional background and mapper examples live in [Concelier SemVer Merge Playbook](merge_semver_playbook.md).
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# Normalized Versions Rollout Dashboard (Sprint 2 – Feedser)
|
||||
# Normalized Versions Rollout Dashboard (Sprint 2 – Concelier)
|
||||
|
||||
_Status date: 2025-10-12 17:05 UTC_
|
||||
|
||||
This dashboard tracks connector readiness for emitting `AffectedPackage.NormalizedVersions` arrays and highlights upcoming coordination checkpoints. Use it alongside:
|
||||
|
||||
- [`src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines.
|
||||
- [Feedser SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes.
|
||||
- [`src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines.
|
||||
- [Concelier SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes.
|
||||
- [Normalized Versions Query Guide](mongo_indices.md) for index/query validation steps.
|
||||
|
||||
## Key milestones
|
||||
@@ -18,28 +18,28 @@ This dashboard tracks connector readiness for emitting `AffectedPackage.Normaliz
|
||||
|
||||
| Connector | Owner team | Normalized versions status | Last update | Next action / link |
|
||||
|-----------|------------|---------------------------|-------------|--------------------|
|
||||
| Acsc | BE-Conn-ACSC | ❌ Not started – mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Feedser.Source.Acsc/TASKS.md`. |
|
||||
| Cccs | BE-Conn-CCCS | ❌ Not started – mapper pending | 2025-10-11 | Add normalized SemVer array in canonical mapper; coordinate fixtures per `TASKS.md`. |
|
||||
| CertBund | BE-Conn-CERTBUND | ✅ Canonical mapper emitting vendor ranges | 2025-10-14 | Normalized vendor range payloads landed alongside telemetry/docs updates; see `src/StellaOps.Feedser.Source.CertBund/TASKS.md`. |
|
||||
| CertCc | BE-Conn-CERTCC | ⚠️ In progress – fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Feedser.Source.CertCc/TASKS.md`. |
|
||||
| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Feedser.Source.Kev.Tests`; `src/StellaOps.Feedser.Source.Kev/TASKS.md`. |
|
||||
| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Feedser.Source.Cve.Tests`); `src/StellaOps.Feedser.Source.Cve/TASKS.md`. |
|
||||
| Ghsa | BE-Conn-GHSA | ⚠️ DOING – normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Feedser.Source.Ghsa/TASKS.md`. |
|
||||
| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Feedser.Source.Osv.Tests`); `src/StellaOps.Feedser.Source.Osv/TASKS.md`. |
|
||||
| Ics.Cisa | BE-Conn-ICS-CISA | ❌ Not started – mapper TODO | 2025-10-11 | Plan SemVer/firmware scheme selection; `src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md`. |
|
||||
| Kisa | BE-Conn-KISA | ✅ Landed 2025-10-14 (mapper + telemetry) | 2025-10-11 | Hangul-aware mapper emits normalized rules; see `docs/dev/kisa_connector_notes.md` for localisation/metric details. |
|
||||
| Ru.Bdu | BE-Conn-BDU | ✅ Raw scheme emitted | 2025-10-14 | Mapper now writes `ru-bdu.raw` normalized rules with provenance + telemetry; `src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md`. |
|
||||
| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started – mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md`. |
|
||||
| Vndr.Apple | BE-Conn-Apple | ✅ Shipped – emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`. |
|
||||
| Vndr.Cisco | BE-Conn-Cisco | ✅ SemVer + vendor extensions emitted | 2025-10-14 | Connector outputs SemVer primitives with `cisco.productId` notes; see `CiscoMapper` and fixtures for coverage. |
|
||||
| Vndr.Msrc | BE-Conn-MSRC | ✅ Map + normalized build rules landed | 2025-10-15 | `MsrcMapper` emits `msrc.build` normalized rules with CVRF references; see `src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md`. |
|
||||
| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up – mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Feedser.Source.Nvd/TASKS.md`. |
|
||||
| Acsc | BE-Conn-ACSC | ❌ Not started – mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Concelier.Connector.Acsc/TASKS.md`. |
|
||||
| Cccs | BE-Conn-CCCS | ⚠️ Scheduled – helper ready, implementation due 2025-10-21 | 2025-10-19 | Apply Merge-provided trailing-version helper to emit `NormalizedVersions`; update mapper/tests per `src/StellaOps.Concelier.Connector.Cccs/TASKS.md`. |
|
||||
| CertBund | BE-Conn-CERTBUND | ⚠️ Follow-up – translate `versions` strings to normalized rules | 2025-10-19 | Build `bis`/`alle` translator + fixtures before 2025-10-22 per `src/StellaOps.Concelier.Connector.CertBund/TASKS.md`. |
|
||||
| CertCc | BE-Conn-CERTCC | ⚠️ In progress – fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Concelier.Connector.CertCc/TASKS.md`. |
|
||||
| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Concelier.Connector.Kev.Tests`; `src/StellaOps.Concelier.Connector.Kev/TASKS.md`. |
|
||||
| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Concelier.Connector.Cve.Tests`); `src/StellaOps.Concelier.Connector.Cve/TASKS.md`. |
|
||||
| Ghsa | BE-Conn-GHSA | ⚠️ DOING – normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Concelier.Connector.Ghsa/TASKS.md`. |
|
||||
| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`); `src/StellaOps.Concelier.Connector.Osv/TASKS.md`. |
|
||||
| Ics.Cisa | BE-Conn-ICS-CISA | ⚠️ Decision pending – normalize SemVer exacts or escalate scheme | 2025-10-19 | Promote `SemVerPrimitive` outputs into `NormalizedVersions` or file Models ticket by 2025-10-23 (`src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md`). |
|
||||
| Kisa | BE-Conn-KISA | ⚠️ Proposal required – firmware scheme due 2025-10-24 | 2025-10-19 | Draft `kisa.build` (or equivalent) scheme with Models, then emit normalized rules; track in `src/StellaOps.Concelier.Connector.Kisa/TASKS.md`. |
|
||||
| Ru.Bdu | BE-Conn-BDU | ✅ Raw scheme emitted | 2025-10-14 | Mapper now writes `ru-bdu.raw` normalized rules with provenance + telemetry; `src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md`. |
|
||||
| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started – mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md`. |
|
||||
| Vndr.Apple | BE-Conn-Apple | ✅ Shipped – emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md`. |
|
||||
| Vndr.Cisco | BE-Conn-Cisco | ⚠️ Scheduled – normalized rule emission due 2025-10-21 | 2025-10-19 | Use Merge helper to persist `NormalizedVersions` alongside SemVer primitives; see `src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md`. |
|
||||
| Vndr.Msrc | BE-Conn-MSRC | ✅ Map + normalized build rules landed | 2025-10-15 | `MsrcMapper` emits `msrc.build` normalized rules with CVRF references; see `src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md`. |
|
||||
| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up – mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Concelier.Connector.Nvd/TASKS.md`. |
|
||||
|
||||
Legend: ✅ complete, ⚠️ in progress/partial, ❌ not started.
|
||||
|
||||
## Monitoring
|
||||
|
||||
- Merge now emits `feedser.merge.normalized_rules` (tags: `package_type`, `scheme`) and `feedser.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out.
|
||||
- Merge now emits `concelier.merge.normalized_rules` (tags: `package_type`, `scheme`) and `concelier.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out.
|
||||
- Expect `normalized_rules_missing` to trend toward zero as each connector flips on normalized output. Investigate any sustained counts by checking the corresponding module `TASKS.md`.
|
||||
|
||||
## Implementation tips
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
id: excititor-my-provider
|
||||
assembly: StellaOps.Excititor.Connectors.MyProvider.dll
|
||||
entryPoint: StellaOps.Excititor.Connectors.MyProvider.MyConnectorPlugin
|
||||
description: |
|
||||
Example connector template. Replace metadata before shipping.
|
||||
tags:
|
||||
- excititor
|
||||
- template
|
||||
@@ -0,0 +1,12 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<!-- Adjust the relative path when copying this template into a repo -->
|
||||
<ProjectReference Include="..\..\..\..\src\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
72
docs/dev/templates/excititor-connector/src/MyConnector.cs
Normal file
72
docs/dev/templates/excititor-connector/src/MyConnector.cs
Normal file
@@ -0,0 +1,72 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.MyProvider;
|
||||
|
||||
public sealed class MyConnector : VexConnectorBase
|
||||
{
|
||||
private readonly IEnumerable<IVexConnectorOptionsValidator<MyConnectorOptions>> _validators;
|
||||
private MyConnectorOptions? _options;
|
||||
|
||||
public MyConnector(VexConnectorDescriptor descriptor, ILogger<MyConnector> logger, TimeProvider timeProvider, IEnumerable<IVexConnectorOptionsValidator<MyConnectorOptions>> validators)
|
||||
: base(descriptor, logger, timeProvider)
|
||||
{
|
||||
_validators = validators;
|
||||
}
|
||||
|
||||
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
_options = VexConnectorOptionsBinder.Bind(
|
||||
Descriptor,
|
||||
settings,
|
||||
validators: _validators);
|
||||
|
||||
LogConnectorEvent(LogLevel.Information, "validate", "MyConnector configuration loaded.",
|
||||
new Dictionary<string, object?>
|
||||
{
|
||||
["catalogUri"] = _options.CatalogUri,
|
||||
["maxParallelRequests"] = _options.MaxParallelRequests,
|
||||
});
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public override IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_options is null)
|
||||
{
|
||||
throw new InvalidOperationException("Connector not validated.");
|
||||
}
|
||||
|
||||
return FetchInternalAsync(context, cancellationToken);
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<VexRawDocument> FetchInternalAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Information, "fetch", "Fetching catalog window...");
|
||||
|
||||
// Replace with real HTTP logic.
|
||||
await Task.Delay(10, cancellationToken);
|
||||
|
||||
var metadata = BuildMetadata(builder => builder
|
||||
.Add("sourceUri", _options!.CatalogUri)
|
||||
.Add("window", context.Since?.ToString("O") ?? "full"));
|
||||
|
||||
yield return CreateRawDocument(
|
||||
VexDocumentFormat.CsafJson,
|
||||
new Uri($"{_options.CatalogUri.TrimEnd('/')}/sample.json"),
|
||||
new byte[] { 0x7B, 0x7D },
|
||||
metadata);
|
||||
}
|
||||
|
||||
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
var claims = ImmutableArray<VexClaim>.Empty;
|
||||
var diagnostics = ImmutableDictionary<string, string>.Empty;
|
||||
return ValueTask.FromResult(new VexClaimBatch(document, claims, diagnostics));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.MyProvider;
|
||||
|
||||
public sealed class MyConnectorOptions
|
||||
{
|
||||
[Required]
|
||||
[Url]
|
||||
public string CatalogUri { get; set; } = default!;
|
||||
|
||||
[Required]
|
||||
public string ApiKey { get; set; } = default!;
|
||||
|
||||
[Range(1, 32)]
|
||||
public int MaxParallelRequests { get; set; } = 4;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.MyProvider;
|
||||
|
||||
public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator<MyConnectorOptions>
|
||||
{
|
||||
public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList<string> errors)
|
||||
{
|
||||
if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
errors.Add("CatalogUri must use HTTPS.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Plugin;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.MyProvider;
|
||||
|
||||
public sealed class MyConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
private static readonly VexConnectorDescriptor Descriptor = new(
|
||||
id: "excititor:my-provider",
|
||||
kind: VexProviderKind.Vendor,
|
||||
displayName: "My Provider VEX");
|
||||
|
||||
public string Name => Descriptor.DisplayName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => true;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
var logger = services.GetRequiredService<ILogger<MyConnector>>();
|
||||
var timeProvider = services.GetRequiredService<TimeProvider>();
|
||||
var validators = services.GetServices<IVexConnectorOptionsValidator<MyConnectorOptions>>();
|
||||
return new MyConnector(Descriptor, logger, timeProvider, validators);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user