feat: Implement approvals workflow and notifications integration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added approvals orchestration with persistence and workflow scaffolding.
- Integrated notifications insights and staged resume hooks.
- Introduced approval coordinator and policy notification bridge with unit tests.
- Added approval decision API with resume requeue and persisted plan snapshots.
- Documented the Excitor consensus API beta and provided JSON sample payload.
- Created analyzers to flag usage of deprecated merge service APIs.
- Implemented logging for artifact uploads and approval decision service.
- Added tests for PackRunApprovalDecisionService and related components.
This commit is contained in:
master
2025-11-06 08:48:13 +02:00
parent 21a2759412
commit dd217b4546
98 changed files with 3883 additions and 2381 deletions

View File

@@ -114,10 +114,10 @@ internal sealed class AdvisoryObservationFactory : IAdvisoryObservationFactory
private static AdvisoryObservationLinkset CreateLinkset(RawIdentifiers identifiers, RawLinkset linkset)
{
var aliases = NormalizeAliases(identifiers, linkset);
var purls = NormalizePackageUrls(linkset.PackageUrls);
var cpes = NormalizeCpes(linkset.Cpes);
var references = NormalizeReferences(linkset.References);
var aliases = CollectAliases(identifiers, linkset);
var purls = CollectValues(linkset.PackageUrls);
var cpes = CollectValues(linkset.Cpes);
var references = CollectReferences(linkset.References);
return new AdvisoryObservationLinkset(aliases, purls, cpes, references);
}
@@ -170,124 +170,91 @@ internal sealed class AdvisoryObservationFactory : IAdvisoryObservationFactory
};
}
private static IEnumerable<string> NormalizeAliases(RawIdentifiers identifiers, RawLinkset linkset)
{
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (LinksetNormalization.TryNormalizeAlias(identifiers.PrimaryId, out var primary))
{
aliases.Add(primary);
}
foreach (var alias in identifiers.Aliases)
{
if (LinksetNormalization.TryNormalizeAlias(alias, out var normalized))
{
aliases.Add(normalized);
}
}
foreach (var alias in linkset.Aliases)
{
if (LinksetNormalization.TryNormalizeAlias(alias, out var normalized))
{
aliases.Add(normalized);
}
}
foreach (var note in linkset.Notes)
{
if (!string.IsNullOrWhiteSpace(note.Value)
&& LinksetNormalization.TryNormalizeAlias(note.Value, out var normalized))
{
aliases.Add(normalized);
}
}
return aliases
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
}
private static IEnumerable<string> NormalizePackageUrls(ImmutableArray<string> packageUrls)
{
if (packageUrls.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var set = new HashSet<string>(StringComparer.Ordinal);
foreach (var candidate in packageUrls)
{
if (!LinksetNormalization.TryNormalizePackageUrl(candidate, out var normalized) || string.IsNullOrEmpty(normalized))
{
continue;
}
set.Add(normalized);
}
return set
.OrderBy(static value => value, StringComparer.Ordinal)
.ToImmutableArray();
}
private static IEnumerable<string> NormalizeCpes(ImmutableArray<string> cpes)
{
if (cpes.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var set = new HashSet<string>(StringComparer.Ordinal);
foreach (var cpe in cpes)
{
if (!LinksetNormalization.TryNormalizeCpe(cpe, out var normalized) || string.IsNullOrEmpty(normalized))
{
continue;
}
set.Add(normalized);
}
return set
.OrderBy(static value => value, StringComparer.Ordinal)
.ToImmutableArray();
}
private static IEnumerable<AdvisoryObservationReference> NormalizeReferences(ImmutableArray<RawReference> references)
{
if (references.IsDefaultOrEmpty)
{
return ImmutableArray<AdvisoryObservationReference>.Empty;
}
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var list = new List<AdvisoryObservationReference>();
foreach (var reference in references)
{
var normalized = LinksetNormalization.TryCreateReference(reference.Type, reference.Url);
if (normalized is null)
{
continue;
}
if (!seen.Add(normalized.Url))
{
continue;
}
list.Add(normalized);
}
return list
.OrderBy(static reference => reference.Type, StringComparer.Ordinal)
.ThenBy(static reference => reference.Url, StringComparer.Ordinal)
.ToImmutableArray();
}
private static IEnumerable<string> CollectAliases(RawIdentifiers identifiers, RawLinkset linkset)
{
var results = new List<string>();
AddAlias(results, identifiers.PrimaryId);
AddRange(results, identifiers.Aliases);
AddRange(results, linkset.Aliases);
foreach (var note in linkset.Notes)
{
if (string.IsNullOrWhiteSpace(note.Value))
{
continue;
}
results.Add(note.Value.Trim());
}
return results;
static void AddAlias(ICollection<string> target, string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
target.Add(value.Trim());
}
static void AddRange(ICollection<string> target, ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return;
}
foreach (var value in values)
{
AddAlias(target, value);
}
}
}
private static IEnumerable<string> CollectValues(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var list = new List<string>(values.Length);
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
list.Add(value.Trim());
}
return list;
}
private static IEnumerable<AdvisoryObservationReference> CollectReferences(ImmutableArray<RawReference> references)
{
if (references.IsDefaultOrEmpty)
{
return ImmutableArray<AdvisoryObservationReference>.Empty;
}
var list = new List<AdvisoryObservationReference>(references.Length);
foreach (var reference in references)
{
if (string.IsNullOrWhiteSpace(reference.Type) || string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
list.Add(new AdvisoryObservationReference(reference.Type.Trim(), reference.Url.Trim()));
}
return list;
}
private static ImmutableDictionary<string, string> CreateAttributes(AdvisoryRawDocument rawDocument)
{

View File

@@ -9,7 +9,7 @@
> Docs alignment (2025-10-26): Linkset expectations detailed in AOC reference §4 and policy-engine architecture §2.1.
> 2025-10-28: Advisory raw ingestion now strips client-supplied supersedes hints, logs ignored pointers, and surfaces repository-supplied supersedes identifiers; service tests cover duplicate handling and append-only semantics.
> Docs alignment (2025-10-26): Deployment guide + observability guide describe supersedes metrics; ensure implementation emits `aoc_violation_total` on failure.
| CONCELIER-CORE-AOC-19-004 `Remove ingestion normalization` | DOING (2025-10-28) | Concelier Core Guild | CONCELIER-CORE-AOC-19-002, POLICY-AOC-19-003 | Strip normalization/dedup/severity logic from ingestion pipelines, delegate derived computations to Policy Engine, and update exporters/tests to consume raw documents only.<br>2025-10-29 19:05Z: Audit completed for `AdvisoryRawService`/Mongo repo to confirm alias order/dedup removal persists; identified remaining normalization in observation/linkset factory that will be revised to surface raw duplicates for Policy ingestion. Change sketch + regression matrix drafted under `docs/dev/aoc-normalization-removal-notes.md` (pending commit).<br>2025-10-31 20:45Z: Added raw linkset projection to observations/storage, exposing canonical+raw views, refreshed fixtures/tests, and documented behaviour in models/doc factory.<br>2025-10-31 21:10Z: Coordinated with Policy Engine (POLICY-ENGINE-20-003) on adoption timeline; backfill + consumer readiness tracked in `docs/dev/raw-linkset-backfill-plan.md`. |
| CONCELIER-CORE-AOC-19-004 `Remove ingestion normalization` | DOING (2025-10-28) | Concelier Core Guild | CONCELIER-CORE-AOC-19-002, POLICY-AOC-19-003 | Strip normalization/dedup/severity logic from ingestion pipelines, delegate derived computations to Policy Engine, and update exporters/tests to consume raw documents only.<br>2025-10-29 19:05Z: Audit completed for `AdvisoryRawService`/Mongo repo to confirm alias order/dedup removal persists; identified remaining normalization in observation/linkset factory that will be revised to surface raw duplicates for Policy ingestion. Change sketch + regression matrix drafted under `docs/dev/aoc-normalization-removal-notes.md` (pending commit).<br>2025-10-31 20:45Z: Added raw linkset projection to observations/storage, exposing canonical+raw views, refreshed fixtures/tests, and documented behaviour in models/doc factory.<br>2025-10-31 21:10Z: Coordinated with Policy Engine (POLICY-ENGINE-20-003) on adoption timeline; backfill + consumer readiness tracked in `docs/dev/raw-linkset-backfill-plan.md`.<br>2025-11-05 14:25Z: Resuming to document merge-dependent normalization paths and prepare implementation notes for `noMergeEnabled` gating before code changes land.<br>2025-11-05 19:20Z: Observation factory/linkset now preserve upstream ordering + duplicates; canonicalisation responsibility shifts to downstream consumers with refreshed unit coverage.<br>2025-11-06 16:10Z: Updated AOC reference/backfill docs with raw vs canonical guidance and cross-linked analyzer guardrails. |
> Docs alignment (2025-10-26): Architecture overview emphasises policy-only derivation; coordinate with Policy Engine guild for rollout.
> 2025-10-29: `AdvisoryRawService` now preserves upstream alias/linkset ordering (trim-only) and updated AOC documentation reflects the behaviour; follow-up to ensure policy consumers handle duplicates remains open.
| CONCELIER-CORE-AOC-19-013 `Authority tenant scope smoke coverage` | TODO | Concelier Core Guild | AUTH-AOC-19-002 | Extend Concelier smoke/e2e fixtures to configure `requiredTenants` and assert cross-tenant rejection with updated Authority tokens. | Coordinate deliverable so Authority docs (`AUTH-AOC-19-003`) can close once tests are in place. |

View File

@@ -5,9 +5,10 @@ using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Merge.Services;
namespace StellaOps.Concelier.Merge.Jobs;
public sealed class MergeReconcileJob : IJob
namespace StellaOps.Concelier.Merge.Jobs;
[Obsolete("MergeReconcileJob is deprecated; Link-Not-Merge supersedes merge scheduling. Disable via concelier:features:noMergeEnabled. Tracking MERGE-LNM-21-002.", DiagnosticId = "CONCELIER0001", UrlFormat = "https://stella-ops.org/docs/migration/no-merge")]
public sealed class MergeReconcileJob : IJob
{
private readonly AdvisoryMergeService _mergeService;
private readonly ILogger<MergeReconcileJob> _logger;

View File

@@ -1,15 +1,17 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core;
using StellaOps.Concelier.Merge.Jobs;
using StellaOps.Concelier.Merge.Jobs;
using StellaOps.Concelier.Merge.Options;
using StellaOps.Concelier.Merge.Services;
namespace StellaOps.Concelier.Merge;
public static class MergeServiceCollectionExtensions
namespace StellaOps.Concelier.Merge;
[Obsolete("Legacy merge module is deprecated; prefer Link-Not-Merge linkset pipelines. Track MERGE-LNM-21-002 and set concelier:features:noMergeEnabled=true to disable registration.", DiagnosticId = "CONCELIER0001", UrlFormat = "https://stella-ops.org/docs/migration/no-merge")]
public static class MergeServiceCollectionExtensions
{
public static IServiceCollection AddMergeModule(this IServiceCollection services, IConfiguration configuration)
{
@@ -34,10 +36,12 @@ public static class MergeServiceCollectionExtensions
return new AdvisoryPrecedenceMerger(resolver, options, timeProvider, logger);
});
services.TryAddSingleton<MergeEventWriter>();
services.TryAddSingleton<AdvisoryMergeService>();
services.AddTransient<MergeReconcileJob>();
return services;
}
}
#pragma warning disable CS0618 // Legacy merge services are marked obsolete.
services.TryAddSingleton<MergeEventWriter>();
services.TryAddSingleton<AdvisoryMergeService>();
services.AddTransient<MergeReconcileJob>();
#pragma warning restore CS0618
return services;
}
}

View File

@@ -14,9 +14,10 @@ using StellaOps.Concelier.Storage.Mongo.Aliases;
using StellaOps.Concelier.Storage.Mongo.MergeEvents;
using System.Text.Json;
namespace StellaOps.Concelier.Merge.Services;
public sealed class AdvisoryMergeService
namespace StellaOps.Concelier.Merge.Services;
[Obsolete("AdvisoryMergeService is deprecated. Transition callers to Link-Not-Merge observation/linkset APIs (MERGE-LNM-21-002) and enable concelier:features:noMergeEnabled when ready.", DiagnosticId = "CONCELIER0001", UrlFormat = "https://stella-ops.org/docs/migration/no-merge")]
public sealed class AdvisoryMergeService
{
private static readonly Meter MergeMeter = new("StellaOps.Concelier.Merge");
private static readonly Counter<long> AliasCollisionCounter = MergeMeter.CreateCounter<long>(

View File

@@ -10,6 +10,6 @@
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|MERGE-LNM-21-001 Migration plan authoring|BE-Merge, Architecture Guild|CONCELIER-LNM-21-101|**DONE (2025-11-03)** Authored `docs/migration/no-merge.md` with rollout phases, backfill/validation checklists, rollback guidance, and ownership matrix for the Link-Not-Merge cutover.|
|MERGE-LNM-21-002 Merge service deprecation|BE-Merge|MERGE-LNM-21-001|**DOING (2025-11-03)** Auditing service registrations, DI bindings, and tests consuming `AdvisoryMergeService`; drafting deprecation plan and analyzer scope prior to code removal.|
|MERGE-LNM-21-002 Merge service deprecation|BE-Merge|MERGE-LNM-21-001|**DOING (2025-11-03)** Auditing service registrations, DI bindings, and tests consuming `AdvisoryMergeService`; drafting deprecation plan and analyzer scope prior to code removal.<br>2025-11-05 14:42Z: Implementing `concelier:features:noMergeEnabled` gate, merge job allowlist checks, `[Obsolete]` markings, and analyzer scaffolding to steer consumers toward linkset APIs.<br>2025-11-06 16:10Z: Introduced Roslyn analyzer (`CONCELIER0002`) referenced by Concelier WebService + tests, documented suppression guidance, and updated migration playbook.|
> 2025-11-03: Catalogued call sites (WebService Program `AddMergeModule`, built-in job registration `merge:reconcile`, `MergeReconcileJob`) and confirmed unit tests are the only direct `MergeAsync` callers; next step is to define analyzer + replacement observability coverage.
|MERGE-LNM-21-003 Determinism/test updates|QA Guild, BE-Merge|MERGE-LNM-21-002|Replace merge determinism suites with observation/linkset regression tests verifying no data mutation and conflicts remain visible.|

View File

@@ -280,57 +280,60 @@ public sealed record AdvisoryObservationLinkset
IEnumerable<string>? cpes,
IEnumerable<AdvisoryObservationReference>? references)
{
Aliases = NormalizeStringSet(aliases, toLower: true);
Purls = NormalizeStringSet(purls);
Cpes = NormalizeStringSet(cpes);
References = NormalizeReferences(references);
}
Aliases = ToImmutableArray(aliases);
Purls = ToImmutableArray(purls);
Cpes = ToImmutableArray(cpes);
References = ToImmutableReferences(references);
}
public ImmutableArray<string> Aliases { get; }
public ImmutableArray<string> Purls { get; }
public ImmutableArray<string> Aliases { get; }
public ImmutableArray<string> Purls { get; }
public ImmutableArray<string> Cpes { get; }
public ImmutableArray<AdvisoryObservationReference> References { get; }
private static ImmutableArray<string> NormalizeStringSet(IEnumerable<string>? values, bool toLower = false)
{
if (values is null)
{
return ImmutableArray<string>.Empty;
}
var list = new List<string>();
foreach (var value in values)
{
var trimmed = Validation.TrimToNull(value);
if (trimmed is null)
{
continue;
}
list.Add(toLower ? trimmed.ToLowerInvariant() : trimmed);
}
return list
.Distinct(StringComparer.Ordinal)
.OrderBy(static v => v, StringComparer.Ordinal)
.ToImmutableArray();
}
private static ImmutableArray<AdvisoryObservationReference> NormalizeReferences(IEnumerable<AdvisoryObservationReference>? references)
{
if (references is null)
{
return ImmutableArray<AdvisoryObservationReference>.Empty;
}
return references
.Where(static reference => reference is not null)
.Distinct()
.OrderBy(static reference => reference.Type, StringComparer.Ordinal)
.ThenBy(static reference => reference.Url, StringComparer.Ordinal)
.ToImmutableArray();
}
}
public ImmutableArray<string> Cpes { get; }
public ImmutableArray<AdvisoryObservationReference> References { get; }
private static ImmutableArray<string> ToImmutableArray(IEnumerable<string>? values)
{
if (values is null)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>();
foreach (var value in values)
{
var trimmed = Validation.TrimToNull(value);
if (trimmed is null)
{
continue;
}
builder.Add(trimmed);
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
private static ImmutableArray<AdvisoryObservationReference> ToImmutableReferences(IEnumerable<AdvisoryObservationReference>? references)
{
if (references is null)
{
return ImmutableArray<AdvisoryObservationReference>.Empty;
}
var builder = ImmutableArray.CreateBuilder<AdvisoryObservationReference>();
foreach (var reference in references)
{
if (reference is null)
{
continue;
}
builder.Add(reference);
}
return builder.Count == 0 ? ImmutableArray<AdvisoryObservationReference>.Empty : builder.ToImmutable();
}
}