Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management

- Added MongoPackRunApprovalStore for managing approval states with MongoDB.
- Introduced MongoPackRunArtifactUploader for uploading and storing artifacts.
- Created MongoPackRunLogStore to handle logging of pack run events.
- Developed MongoPackRunStateStore for persisting and retrieving pack run states.
- Implemented unit tests for MongoDB stores to ensure correct functionality.
- Added MongoTaskRunnerTestContext for setting up MongoDB test environment.
- Enhanced PackRunStateFactory to correctly initialize state with gate reasons.

src/Concelier/StellaOps.Concelier.WebService/Contracts/AdvisoryChunkResponses.cs

@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+
+namespace StellaOps.Concelier.WebService.Contracts;
+
+public sealed record AdvisoryChunkCollectionResponse(
+    string AdvisoryKey,
+    int Total,
+    bool Truncated,
+    IReadOnlyList<AdvisoryChunkItemResponse> Chunks,
+    IReadOnlyList<AdvisoryChunkSourceResponse> Sources);
+
+public sealed record AdvisoryChunkItemResponse(
+    string DocumentId,
+    string ChunkId,
+    string Section,
+    string ParagraphId,
+    string Text,
+    IReadOnlyDictionary<string, string> Metadata);
+
+public sealed record AdvisoryChunkSourceResponse(
+    string ObservationId,
+    string DocumentId,
+    string Format,
+    string Vendor,
+    string ContentHash,
+    DateTimeOffset CreatedAt);

src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs

@@ -111,14 +111,17 @@ internal static class JobRegistrationExtensions
 
     private static void ConfigureMergeJob(JobSchedulerOptions options, IConfiguration configuration)
     {
-        var noMergeEnabled = configuration.GetValue<bool?>("concelier:features:noMergeEnabled") ?? true;
+        var noMergeEnabled = configuration.GetValue<bool?>("concelier:features:noMergeEnabled")
+            ?? configuration.GetValue<bool?>("features:noMergeEnabled")
+            ?? true;
         if (noMergeEnabled)
         {
             options.Definitions.Remove(MergeReconcileBuiltInJob.Kind);
             return;
         }
 
-        var allowlist = configuration.GetSection("concelier:jobs:merge:allowlist").Get<string[]>();
+        var allowlist = configuration.GetSection("concelier:jobs:merge:allowlist").Get<string[]>()
+            ?? configuration.GetSection("jobs:merge:allowlist").Get<string[]>();
         if (allowlist is { Length: > 0 })
         {
             var allowlistSet = new HashSet<string>(allowlist, StringComparer.OrdinalIgnoreCase);

src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs

@@ -17,6 +17,8 @@ public sealed class ConcelierOptions
     public MirrorOptions Mirror { get; set; } = new();
 
     public FeaturesOptions Features { get; set; } = new();
+
+    public AdvisoryChunkOptions AdvisoryChunks { get; set; } = new();
 
     public sealed class StorageOptions
     {
@@ -81,6 +83,8 @@ public sealed class ConcelierOptions
 
         public IList<string> RequiredScopes { get; set; } = new List<string>();
 
+        public IList<string> RequiredTenants { get; set; } = new List<string>();
+
         public IList<string> BypassNetworks { get; set; } = new List<string>();
 
         public string? ClientId { get; set; }
@@ -146,4 +150,19 @@ public sealed class ConcelierOptions
 
         public IList<string> MergeJobAllowlist { get; } = new List<string>();
     }
+
+    public sealed class AdvisoryChunkOptions
+    {
+        public int DefaultChunkLimit { get; set; } = 200;
+
+        public int MaxChunkLimit { get; set; } = 400;
+
+        public int DefaultObservationLimit { get; set; } = 24;
+
+        public int MaxObservationLimit { get; set; } = 48;
+
+        public int DefaultMinimumLength { get; set; } = 64;
+
+        public int MaxMinimumLength { get; set; } = 512;
+    }
 }

src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs

@@ -30,11 +30,14 @@ public static class ConcelierOptionsValidator
 
         options.Authority ??= new ConcelierOptions.AuthorityOptions();
         options.Authority.Resilience ??= new ConcelierOptions.AuthorityOptions.ResilienceOptions();
+        options.Authority.RequiredTenants ??= new List<string>();
         NormalizeList(options.Authority.Audiences, toLower: false);
         NormalizeList(options.Authority.RequiredScopes, toLower: true);
         NormalizeList(options.Authority.BypassNetworks, toLower: false);
         NormalizeList(options.Authority.ClientScopes, toLower: true);
+        NormalizeList(options.Authority.RequiredTenants, toLower: true);
         ValidateResilience(options.Authority.Resilience);
+        ValidateTenantAllowlist(options.Authority.RequiredTenants);
 
         if (options.Authority.RequiredScopes.Count == 0)
         {
@@ -133,6 +136,9 @@ public static class ConcelierOptionsValidator
 
         options.Mirror ??= new ConcelierOptions.MirrorOptions();
         ValidateMirror(options.Mirror);
+
+        options.AdvisoryChunks ??= new ConcelierOptions.AdvisoryChunkOptions();
+        ValidateAdvisoryChunks(options.AdvisoryChunks);
     }
 
     private static void NormalizeList(IList<string> values, bool toLower)
@@ -190,6 +196,32 @@ public static class ConcelierOptionsValidator
         }
     }
 
+    private static void ValidateTenantAllowlist(IList<string> tenants)
+    {
+        if (tenants is null || tenants.Count == 0)
+        {
+            return;
+        }
+
+        foreach (var tenant in tenants)
+        {
+            if (string.IsNullOrEmpty(tenant) || tenant.Length > 64)
+            {
+                throw new InvalidOperationException("Authority requiredTenants entries must be between 1 and 64 characters.");
+            }
+
+            foreach (var ch in tenant)
+            {
+                var isAlpha = ch is >= 'a' and <= 'z';
+                var isDigit = ch is >= '0' and <= '9';
+                if (!isAlpha && !isDigit && ch != '-')
+                {
+                    throw new InvalidOperationException($"Authority requiredTenants entry '{tenant}' contains invalid character '{ch}'. Use lowercase letters, digits, or '-'.");
+                }
+            }
+        }
+    }
+
     private static void ValidateMirror(ConcelierOptions.MirrorOptions mirror)
     {
         if (mirror.MaxIndexRequestsPerHour < 0)
@@ -242,4 +274,37 @@ public static class ConcelierOptionsValidator
             throw new InvalidOperationException("Mirror distribution requires at least one domain when enabled.");
         }
     }
+
+    private static void ValidateAdvisoryChunks(ConcelierOptions.AdvisoryChunkOptions chunks)
+    {
+        if (chunks.DefaultChunkLimit <= 0)
+        {
+            throw new InvalidOperationException("Advisory chunk defaultChunkLimit must be greater than zero.");
+        }
+
+        if (chunks.MaxChunkLimit < chunks.DefaultChunkLimit)
+        {
+            throw new InvalidOperationException("Advisory chunk maxChunkLimit must be greater than or equal to defaultChunkLimit.");
+        }
+
+        if (chunks.DefaultObservationLimit <= 0)
+        {
+            throw new InvalidOperationException("Advisory chunk defaultObservationLimit must be greater than zero.");
+        }
+
+        if (chunks.MaxObservationLimit < chunks.DefaultObservationLimit)
+        {
+            throw new InvalidOperationException("Advisory chunk maxObservationLimit must be greater than or equal to defaultObservationLimit.");
+        }
+
+        if (chunks.DefaultMinimumLength <= 0)
+        {
+            throw new InvalidOperationException("Advisory chunk defaultMinimumLength must be greater than zero.");
+        }
+
+        if (chunks.MaxMinimumLength < chunks.DefaultMinimumLength)
+        {
+            throw new InvalidOperationException("Advisory chunk maxMinimumLength must be greater than or equal to defaultMinimumLength.");
+        }
+    }
 }

src/Concelier/StellaOps.Concelier.WebService/Program.cs

@@ -235,6 +235,12 @@ var resolvedConcelierOptions = app.Services.GetRequiredService<IOptions<Concelie
 var resolvedAuthority = resolvedConcelierOptions.Authority ?? new ConcelierOptions.AuthorityOptions();
 authorityConfigured = resolvedAuthority.Enabled;
 var enforceAuthority = resolvedAuthority.Enabled && !resolvedAuthority.AllowAnonymousFallback;
+var requiredTenants = (resolvedAuthority.RequiredTenants ?? Array.Empty<string>())
+    .Select(static tenant => tenant?.Trim().ToLowerInvariant())
+    .Where(static tenant => !string.IsNullOrWhiteSpace(tenant))
+    .Distinct(StringComparer.Ordinal)
+    .ToImmutableHashSet(StringComparer.Ordinal);
+var enforceTenantAllowlist = !requiredTenants.IsEmpty;
 
 if (resolvedAuthority.Enabled && resolvedAuthority.AllowAnonymousFallback)
 {
@@ -358,11 +364,14 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
     AdvisoryIngestRequest request,
     [FromServices] IAdvisoryRawService rawService,
     [FromServices] TimeProvider timeProvider,
+    [FromServices] ILogger<Program> logger,
     CancellationToken cancellationToken) =>
 {
     ApplyNoCache(context.Response);
 
-    if (request is null || request.Source is null || request.Upstream is null || request.Content is null || request.Identifiers is null)
+    var ingestRequest = request;
+
+    if (ingestRequest is null || ingestRequest.Source is null || ingestRequest.Upstream is null || ingestRequest.Content is null || ingestRequest.Identifiers is null)
     {
         return Problem(context, "Invalid request", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "source, upstream, content, and identifiers sections are required.");
     }
@@ -381,7 +390,14 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
     AdvisoryRawDocument document;
     try
     {
-        document = AdvisoryRawRequestMapper.Map(request, tenant, timeProvider);
+        logger.LogWarning(
+            "Binding advisory ingest request hash={Hash}",
+            ingestRequest.Upstream.ContentHash ?? "(null)");
+
+        document = AdvisoryRawRequestMapper.Map(ingestRequest, tenant, timeProvider);
+        logger.LogWarning(
+            "Mapped advisory_raw document hash={Hash}",
+            string.IsNullOrWhiteSpace(document.Upstream.ContentHash) ? "(empty)" : document.Upstream.ContentHash);
     }
     catch (Exception ex) when (ex is ArgumentException or InvalidOperationException)
     {
@@ -418,6 +434,15 @@ var advisoryIngestEndpoint = app.MapPost("/ingest/advisory", async (
     }
     catch (ConcelierAocGuardException guardException)
     {
+        logger.LogWarning(
+            guardException,
+            "AOC guard rejected advisory ingest tenant={Tenant} upstream={UpstreamId} requestHash={RequestHash} documentHash={DocumentHash} codes={Codes}",
+            tenant,
+            document.Upstream.UpstreamId,
+            request!.Upstream?.ContentHash ?? "(null)",
+            string.IsNullOrWhiteSpace(document.Upstream.ContentHash) ? "(empty)" : document.Upstream.ContentHash,
+            string.Join(',', guardException.Violations.Select(static violation => violation.ErrorCode)));
+
         IngestionMetrics.ViolationCounter.Add(1, new[]
         {
             new KeyValuePair<string, object?>("tenant", tenant),
@@ -945,6 +970,11 @@ IResult? EnsureTenantAuthorized(HttpContext context, string tenant)
         return null;
     }
 
+    if (enforceTenantAllowlist && !requiredTenants.Contains(tenant))
+    {
+        return Results.Forbid();
+    }
+
     var principal = context.User;
 
     if (enforceAuthority && (principal?.Identity?.IsAuthenticated != true))
@@ -965,6 +995,11 @@ IResult? EnsureTenantAuthorized(HttpContext context, string tenant)
         {
             return Results.Forbid();
         }
+
+        if (enforceTenantAllowlist && !requiredTenants.Contains(normalizedClaim))
+        {
+            return Results.Forbid();
+        }
     }
 
     return null;

src/Concelier/StellaOps.Concelier.WebService/Services/AdvisoryChunkBuilder.cs

@@ -0,0 +1,257 @@
+using System.Collections.Immutable;
+using System.Globalization;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using StellaOps.Concelier.Models.Observations;
+using StellaOps.Concelier.WebService.Contracts;
+
+namespace StellaOps.Concelier.WebService.Services;
+
+internal sealed record AdvisoryChunkBuildOptions(
+    string AdvisoryKey,
+    int ChunkLimit,
+    int ObservationLimit,
+    ImmutableHashSet<string> SectionFilter,
+    ImmutableHashSet<string> FormatFilter,
+    int MinimumLength);
+
+internal sealed class AdvisoryChunkBuilder
+{
+    private const int DefaultMinLength = 40;
+
+    public AdvisoryChunkCollectionResponse Build(
+        AdvisoryChunkBuildOptions options,
+        IReadOnlyList<AdvisoryObservation> observations)
+    {
+        var chunks = new List<AdvisoryChunkItemResponse>(Math.Min(options.ChunkLimit, 256));
+        var sources = new List<AdvisoryChunkSourceResponse>();
+        var total = 0;
+        var truncated = false;
+
+        foreach (var observation in observations
+            .OrderByDescending(o => o.CreatedAt))
+        {
+            if (sources.Count >= options.ObservationLimit)
+            {
+                truncated = truncated || chunks.Count == options.ChunkLimit;
+                break;
+            }
+
+            if (options.FormatFilter.Count > 0 &&
+                !options.FormatFilter.Contains(observation.Content.Format))
+            {
+                continue;
+            }
+
+            var documentId = DetermineDocumentId(observation);
+            sources.Add(new AdvisoryChunkSourceResponse(
+                observation.ObservationId,
+                documentId,
+                observation.Content.Format,
+                observation.Source.Vendor,
+                observation.Upstream.ContentHash,
+                observation.CreatedAt));
+
+            foreach (var chunk in ExtractChunks(observation, documentId, options))
+            {
+                total++;
+                if (chunks.Count < options.ChunkLimit)
+                {
+                    chunks.Add(chunk);
+                }
+                else
+                {
+                    truncated = true;
+                    break;
+                }
+            }
+
+            if (truncated)
+            {
+                break;
+            }
+        }
+
+        if (!truncated)
+        {
+            total = chunks.Count;
+        }
+
+        return new AdvisoryChunkCollectionResponse(
+            options.AdvisoryKey,
+            total,
+            truncated,
+            chunks,
+            sources);
+    }
+
+    private static string DetermineDocumentId(AdvisoryObservation observation)
+    {
+        if (!string.IsNullOrWhiteSpace(observation.Upstream.UpstreamId))
+        {
+            return observation.Upstream.UpstreamId;
+        }
+
+        return observation.ObservationId;
+    }
+
+    private static IEnumerable<AdvisoryChunkItemResponse> ExtractChunks(
+        AdvisoryObservation observation,
+        string documentId,
+        AdvisoryChunkBuildOptions options)
+    {
+        var root = observation.Content.Raw;
+        if (root is null)
+        {
+            yield break;
+        }
+
+        var stack = new Stack<(JsonNode Node, string Path, string Section)>();
+        stack.Push((root, string.Empty, string.Empty));
+
+        while (stack.Count > 0)
+        {
+            var (node, path, section) = stack.Pop();
+            if (node is null)
+            {
+                continue;
+            }
+
+            switch (node)
+            {
+                case JsonValue value when TryNormalize(value, out var text):
+                    if (text.Length < Math.Max(options.MinimumLength, DefaultMinLength))
+                    {
+                        continue;
+                    }
+
+                    if (!ContainsLetter(text))
+                    {
+                        continue;
+                    }
+
+                    var resolvedSection = string.IsNullOrEmpty(section) ? documentId : section;
+                    if (options.SectionFilter.Count > 0 && !options.SectionFilter.Contains(resolvedSection))
+                    {
+                        continue;
+                    }
+
+                    var paragraphId = string.IsNullOrEmpty(path) ? resolvedSection : path;
+                    var chunkId = CreateChunkId(documentId, paragraphId);
+                    var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
+                    {
+                        ["path"] = paragraphId,
+                        ["section"] = resolvedSection,
+                        ["format"] = observation.Content.Format
+                    };
+
+                    if (!string.IsNullOrEmpty(observation.Content.SpecVersion))
+                    {
+                        metadata["specVersion"] = observation.Content.SpecVersion!;
+                    }
+
+                    yield return new AdvisoryChunkItemResponse(
+                        documentId,
+                        chunkId,
+                        resolvedSection,
+                        paragraphId,
+                        text,
+                        metadata);
+                    break;
+
+                case JsonObject obj:
+                    foreach (var property in obj.Reverse())
+                    {
+                        var childSection = string.IsNullOrEmpty(section) ? property.Key : section;
+                        var childPath = AppendPath(path, property.Key);
+                        if (property.Value is { } childNode)
+                        {
+                            stack.Push((childNode, childPath, childSection));
+                        }
+                    }
+
+                    break;
+
+                case JsonArray array:
+                    for (var index = array.Count - 1; index >= 0; index--)
+                    {
+                        var childPath = AppendIndex(path, index);
+                        if (array[index] is { } childNode)
+                        {
+                            stack.Push((childNode, childPath, section));
+                        }
+                    }
+
+                    break;
+            }
+        }
+    }
+
+    private static bool TryNormalize(JsonValue value, out string normalized)
+    {
+        normalized = string.Empty;
+        if (!value.TryGetValue(out string? text) || text is null)
+        {
+            return false;
+        }
+
+        var span = text.AsSpan();
+        var builder = new StringBuilder(span.Length);
+        var previousWhitespace = false;
+
+        foreach (var ch in span)
+        {
+            if (char.IsControl(ch) && !char.IsWhiteSpace(ch))
+            {
+                continue;
+            }
+
+            if (char.IsWhiteSpace(ch))
+            {
+                if (previousWhitespace)
+                {
+                    continue;
+                }
+
+                builder.Append(' ');
+                previousWhitespace = true;
+            }
+            else
+            {
+                builder.Append(ch);
+                previousWhitespace = false;
+            }
+        }
+
+        normalized = builder.ToString().Trim();
+        return normalized.Length > 0;
+    }
+
+    private static bool ContainsLetter(string text)
+        => text.Any(static ch => char.IsLetter(ch));
+
+    private static string AppendPath(string path, string? segment)
+    {
+        var safeSegment = segment ?? string.Empty;
+        return string.IsNullOrEmpty(path) ? safeSegment : string.Concat(path, '.', safeSegment);
+    }
+
+    private static string AppendIndex(string path, int index)
+    {
+        if (string.IsNullOrEmpty(path))
+        {
+            return $"[{index}]";
+        }
+
+        return string.Concat(path, '[', index.ToString(CultureInfo.InvariantCulture), ']');
+    }
+
+    private static string CreateChunkId(string documentId, string paragraphId)
+    {
+        var input = string.Concat(documentId, '|', paragraphId);
+        var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input));
+        return string.Concat(documentId, ':', Convert.ToHexString(hash.AsSpan(0, 8)));
+    }
+}

src/Concelier/StellaOps.Concelier.WebService/TASKS.md

@@ -63,7 +63,7 @@
 
 | ID | Status | Owner(s) | Depends on | Notes |
 |----|--------|----------|------------|-------|
-| CONCELIER-AIAI-31-001 `Paragraph anchors` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001 | Expose advisory chunk API returning paragraph anchors, section metadata, and token-safe text for Advisory AI retrieval. |
+| CONCELIER-AIAI-31-001 `Paragraph anchors` | DONE | Concelier WebService Guild | CONCELIER-VULN-29-001 | Expose advisory chunk API returning paragraph anchors, section metadata, and token-safe text for Advisory AI retrieval. See docs/updates/2025-11-07-concelier-advisory-chunks.md. |
 | CONCELIER-AIAI-31-002 `Structured fields` | TODO | Concelier WebService Guild | CONCELIER-AIAI-31-001 | Ensure observation APIs expose upstream workaround/fix/CVSS fields with provenance; add caching for summary queries. |
 | CONCELIER-AIAI-31-003 `Advisory AI telemetry` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-AIAI-31-001 | Emit metrics/logs for chunk requests, cache hits, and guardrail blocks triggered by advisory payloads. |