Search/AdvisoryAI and DAL conversion to EF finishes up. Preparation for microservices consolidation.

src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchAnalyticsEndpoints.cs

@@ -16,7 +16,8 @@ public static class SearchAnalyticsEndpoints
     {
         "query",
         "click",
-        "zero_result"
+        "zero_result",
+        "synthesis"
     };
 
     public static RouteGroupBuilder MapSearchAnalyticsEndpoints(this IEndpointRouteBuilder builder)
@@ -29,10 +30,10 @@ public static class SearchAnalyticsEndpoints
 
         group.MapPost("/analytics", RecordAnalyticsAsync)
             .WithName("SearchAnalyticsRecord")
-            .WithSummary("Records batch search analytics events (query, click, zero_result).")
+            .WithSummary("Records batch search analytics events (query, click, zero_result, synthesis).")
             .WithDescription(
                 "Accepts a batch of search analytics events for tracking query frequency, click-through rates, " +
-                "and zero-result queries. Events are tenant-scoped and user ID is optional for privacy. " +
+                "zero-result queries, and synthesis usage. Queries and user identifiers are pseudonymized before persistence. " +
                 "Fire-and-forget from the client; failures do not affect search functionality.")
             .RequireAuthorization(AdvisoryAIPolicies.OperatePolicy)
             .Produces(StatusCodes.Status204NoContent)

src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchFeedbackEndpoints.cs

@@ -29,7 +29,7 @@ public static class SearchFeedbackEndpoints
             .WithSummary("Submits user feedback (helpful/not_helpful) for a search result or synthesis.")
             .WithDescription(
                 "Records a thumbs-up or thumbs-down signal for a specific search result, " +
-                "identified by entity key and domain. Used to improve search quality over time. " +
+                "identified by entity key and domain. Query/user dimensions are pseudonymized for analytics privacy. " +
                 "Fire-and-forget from the UI perspective.")
             .RequireAuthorization(AdvisoryAIPolicies.ViewPolicy)
             .Produces(StatusCodes.Status201Created)
@@ -199,6 +199,34 @@ public static class SearchFeedbackEndpoints
             AvgResultCount = metrics.AvgResultCount,
             FeedbackScore = metrics.FeedbackScore,
             Period = metrics.Period,
+            LowQualityResults = metrics.LowQualityResults
+                .Select(row => new SearchLowQualityResultDto
+                {
+                    EntityKey = row.EntityKey,
+                    Domain = row.Domain,
+                    NegativeFeedbackCount = row.NegativeFeedbackCount,
+                    TotalFeedback = row.TotalFeedback,
+                    NegativeRate = row.NegativeRate,
+                })
+                .ToArray(),
+            TopQueries = metrics.TopQueries
+                .Select(row => new SearchTopQueryDto
+                {
+                    Query = row.Query,
+                    TotalSearches = row.TotalSearches,
+                    AvgResultCount = row.AvgResultCount,
+                    FeedbackScore = row.FeedbackScore,
+                })
+                .ToArray(),
+            Trend = metrics.Trend
+                .Select(point => new SearchQualityTrendPointDto
+                {
+                    Day = point.Day.ToString("yyyy-MM-dd"),
+                    TotalSearches = point.TotalSearches,
+                    ZeroResultRate = point.ZeroResultRate,
+                    FeedbackScore = point.FeedbackScore,
+                })
+                .ToArray(),
         });
     }
 
@@ -281,4 +309,32 @@ public sealed record SearchQualityMetricsDto
     public double AvgResultCount { get; init; }
     public double FeedbackScore { get; init; }
     public string Period { get; init; } = "7d";
+    public IReadOnlyList<SearchLowQualityResultDto> LowQualityResults { get; init; } = [];
+    public IReadOnlyList<SearchTopQueryDto> TopQueries { get; init; } = [];
+    public IReadOnlyList<SearchQualityTrendPointDto> Trend { get; init; } = [];
+}
+
+public sealed record SearchLowQualityResultDto
+{
+    public string EntityKey { get; init; } = string.Empty;
+    public string Domain { get; init; } = string.Empty;
+    public int NegativeFeedbackCount { get; init; }
+    public int TotalFeedback { get; init; }
+    public double NegativeRate { get; init; }
+}
+
+public sealed record SearchTopQueryDto
+{
+    public string Query { get; init; } = string.Empty;
+    public int TotalSearches { get; init; }
+    public double AvgResultCount { get; init; }
+    public double FeedbackScore { get; init; }
+}
+
+public sealed record SearchQualityTrendPointDto
+{
+    public string Day { get; init; } = string.Empty;
+    public int TotalSearches { get; init; }
+    public double ZeroResultRate { get; init; }
+    public double FeedbackScore { get; init; }
 }

src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/UnifiedSearchEndpoints.cs

@@ -2,9 +2,11 @@ using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing;
 using StellaOps.AdvisoryAI.UnifiedSearch;
+using StellaOps.AdvisoryAI.UnifiedSearch.Synthesis;
 using StellaOps.AdvisoryAI.WebService.Security;
 using StellaOps.Auth.ServerIntegration.Tenancy;
 using System.Linq;
+using System.Text.Json;
 using static StellaOps.Localization.T;
 
 namespace StellaOps.AdvisoryAI.WebService.Endpoints;
@@ -17,7 +19,11 @@ public static class UnifiedSearchEndpoints
         "findings",
         "vex",
         "policy",
-        "platform"
+        "platform",
+        "graph",
+        "timeline",
+        "scanner",
+        "opsmemory"
     };
 
     private static readonly HashSet<string> AllowedEntityTypes = new(StringComparer.Ordinal)
@@ -28,7 +34,13 @@ public static class UnifiedSearchEndpoints
         "finding",
         "vex_statement",
         "policy_rule",
-        "platform_entity"
+        "platform_entity",
+        "package",
+        "image",
+        "registry",
+        "event",
+        "scan",
+        "graph_node"
     };
 
     public static RouteGroupBuilder MapUnifiedSearchEndpoints(this IEndpointRouteBuilder builder)
@@ -51,6 +63,17 @@ public static class UnifiedSearchEndpoints
             .Produces(StatusCodes.Status400BadRequest)
             .Produces(StatusCodes.Status403Forbidden);
 
+        group.MapPost("/synthesize", SynthesizeAsync)
+            .WithName("UnifiedSearchSynthesize")
+            .WithSummary("Streams deterministic-first search synthesis as SSE.")
+            .WithDescription(
+                "Produces deterministic synthesis first, then optional LLM synthesis chunks, grounding score, and actions. " +
+                "Requires search synthesis scope and tenant context.")
+            .RequireAuthorization(AdvisoryAIPolicies.OperatePolicy)
+            .Produces(StatusCodes.Status200OK, contentType: "text/event-stream")
+            .Produces(StatusCodes.Status400BadRequest)
+            .Produces(StatusCodes.Status403Forbidden);
+
         group.MapPost("/index/rebuild", RebuildIndexAsync)
             .WithName("UnifiedSearchRebuild")
             .WithSummary("Rebuilds unified search index from configured ingestion sources.")
@@ -90,12 +113,14 @@ public static class UnifiedSearchEndpoints
         try
         {
             var userScopes = ResolveUserScopes(httpContext);
+            var userId = ResolveUserId(httpContext);
             var domainRequest = new UnifiedSearchRequest(
                 request.Q.Trim(),
                 request.K,
-                NormalizeFilter(request.Filters, tenant, userScopes),
+                NormalizeFilter(request.Filters, tenant, userScopes, userId),
                 request.IncludeSynthesis,
-                request.IncludeDebug);
+                request.IncludeDebug,
+                NormalizeAmbient(request.Ambient));
 
             var response = await searchService.SearchAsync(domainRequest, cancellationToken).ConfigureAwait(false);
             return Results.Ok(MapResponse(response));
@@ -106,6 +131,158 @@ public static class UnifiedSearchEndpoints
         }
     }
 
+    private static async Task SynthesizeAsync(
+        HttpContext httpContext,
+        UnifiedSearchSynthesizeApiRequest request,
+        SearchSynthesisService synthesisService,
+        CancellationToken cancellationToken)
+    {
+        if (request is null || string.IsNullOrWhiteSpace(request.Q))
+        {
+            httpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
+            await httpContext.Response.WriteAsJsonAsync(new { error = _t("advisoryai.validation.q_required") }, cancellationToken);
+            return;
+        }
+
+        var tenant = ResolveTenant(httpContext);
+        if (tenant is null)
+        {
+            httpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
+            await httpContext.Response.WriteAsJsonAsync(new { error = _t("advisoryai.validation.tenant_required") }, cancellationToken);
+            return;
+        }
+
+        if (!HasSynthesisScope(httpContext))
+        {
+            httpContext.Response.StatusCode = StatusCodes.Status403Forbidden;
+            await httpContext.Response.WriteAsJsonAsync(new { error = "Missing required scope: search:synthesize" }, cancellationToken);
+            return;
+        }
+
+        var cards = MapSynthesisCards(request.TopCards);
+        if (cards.Count == 0)
+        {
+            httpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
+            await httpContext.Response.WriteAsJsonAsync(new { error = "topCards is required" }, cancellationToken);
+            return;
+        }
+
+        var userId = ResolveUserId(httpContext) ?? "anonymous";
+        var domainRequest = new SearchSynthesisRequest(
+            request.Q.Trim(),
+            cards,
+            request.Plan,
+            request.Preferences is null
+                ? null
+                : new SearchSynthesisPreferences
+                {
+                    Depth = request.Preferences.Depth,
+                    MaxTokens = request.Preferences.MaxTokens,
+                    IncludeActions = request.Preferences.IncludeActions,
+                    Locale = request.Preferences.Locale
+                });
+
+        httpContext.Response.StatusCode = StatusCodes.Status200OK;
+        httpContext.Response.ContentType = "text/event-stream";
+        httpContext.Response.Headers.CacheControl = "no-cache";
+        httpContext.Response.Headers.Connection = "keep-alive";
+        await httpContext.Response.Body.FlushAsync(cancellationToken);
+
+        var started = DateTimeOffset.UtcNow;
+        try
+        {
+            var result = await synthesisService.ExecuteAsync(
+                tenant,
+                userId,
+                domainRequest,
+                cancellationToken).ConfigureAwait(false);
+
+            await WriteSseEventAsync(httpContext, "synthesis_start", new
+            {
+                tier = "deterministic",
+                summary = result.DeterministicSummary
+            }, cancellationToken).ConfigureAwait(false);
+
+            if (result.QuotaExceeded)
+            {
+                await WriteSseEventAsync(httpContext, "llm_status", new { status = "quota_exceeded" }, cancellationToken).ConfigureAwait(false);
+            }
+            else if (result.LlmUnavailable || string.IsNullOrWhiteSpace(result.LlmSummary))
+            {
+                await WriteSseEventAsync(httpContext, "llm_status", new { status = "unavailable" }, cancellationToken).ConfigureAwait(false);
+            }
+            else
+            {
+                await WriteSseEventAsync(httpContext, "llm_status", new { status = "starting" }, cancellationToken).ConfigureAwait(false);
+                await WriteSseEventAsync(httpContext, "llm_status", new { status = "streaming" }, cancellationToken).ConfigureAwait(false);
+
+                foreach (var chunk in Chunk(result.LlmSummary, 240))
+                {
+                    await WriteSseEventAsync(httpContext, "llm_chunk", new
+                    {
+                        content = chunk,
+                        isComplete = false
+                    }, cancellationToken).ConfigureAwait(false);
+                }
+
+                await WriteSseEventAsync(httpContext, "llm_chunk", new
+                {
+                    content = string.Empty,
+                    isComplete = true
+                }, cancellationToken).ConfigureAwait(false);
+
+                await WriteSseEventAsync(httpContext, "llm_status", new { status = "validating" }, cancellationToken).ConfigureAwait(false);
+                await WriteSseEventAsync(httpContext, "grounding", new
+                {
+                    score = result.GroundingScore,
+                    citations = 0,
+                    ungrounded = 0,
+                    issues = Array.Empty<string>()
+                }, cancellationToken).ConfigureAwait(false);
+                await WriteSseEventAsync(httpContext, "llm_status", new { status = "complete" }, cancellationToken).ConfigureAwait(false);
+            }
+
+            if (result.Actions.Count > 0)
+            {
+                await WriteSseEventAsync(httpContext, "actions", new
+                {
+                    actions = result.Actions.Select(static action => new
+                    {
+                        label = action.Label,
+                        route = action.Route,
+                        sourceEntityKey = action.SourceEntityKey
+                    })
+                }, cancellationToken).ConfigureAwait(false);
+            }
+
+            var durationMs = (long)(DateTimeOffset.UtcNow - started).TotalMilliseconds;
+            await WriteSseEventAsync(httpContext, "synthesis_end", new
+            {
+                totalTokens = result.TotalTokens,
+                durationMs,
+                provider = result.Provider,
+                promptVersion = result.PromptVersion
+            }, cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            await WriteSseEventAsync(httpContext, "error", new
+            {
+                code = "synthesis_error",
+                message = ex.Message
+            }, cancellationToken).ConfigureAwait(false);
+
+            var durationMs = (long)(DateTimeOffset.UtcNow - started).TotalMilliseconds;
+            await WriteSseEventAsync(httpContext, "synthesis_end", new
+            {
+                totalTokens = 0,
+                durationMs,
+                provider = "none",
+                promptVersion = "search-synth-v1"
+            }, cancellationToken).ConfigureAwait(false);
+        }
+    }
+
     private static async Task<IResult> RebuildIndexAsync(
         HttpContext httpContext,
         IUnifiedSearchIndexer indexer,
@@ -125,14 +302,19 @@ public static class UnifiedSearchEndpoints
         });
     }
 
-    private static UnifiedSearchFilter? NormalizeFilter(UnifiedSearchApiFilter? filter, string tenant, IReadOnlyList<string>? userScopes = null)
+    private static UnifiedSearchFilter? NormalizeFilter(
+        UnifiedSearchApiFilter? filter,
+        string tenant,
+        IReadOnlyList<string>? userScopes = null,
+        string? userId = null)
     {
         if (filter is null)
         {
             return new UnifiedSearchFilter
             {
                 Tenant = tenant,
-                UserScopes = userScopes
+                UserScopes = userScopes,
+                UserId = userId
             };
         }
 
@@ -180,7 +362,37 @@ public static class UnifiedSearchEndpoints
             Service = string.IsNullOrWhiteSpace(filter.Service) ? null : filter.Service.Trim(),
             Tags = tags,
             Tenant = tenant,
-            UserScopes = userScopes
+            UserScopes = userScopes,
+            UserId = userId
+        };
+    }
+
+    private static AmbientContext? NormalizeAmbient(UnifiedSearchApiAmbientContext? ambient)
+    {
+        if (ambient is null)
+        {
+            return null;
+        }
+
+        return new AmbientContext
+        {
+            CurrentRoute = string.IsNullOrWhiteSpace(ambient.CurrentRoute) ? null : ambient.CurrentRoute.Trim(),
+            SessionId = string.IsNullOrWhiteSpace(ambient.SessionId) ? null : ambient.SessionId.Trim(),
+            ResetSession = ambient.ResetSession,
+            VisibleEntityKeys = ambient.VisibleEntityKeys is { Count: > 0 }
+                ? ambient.VisibleEntityKeys
+                    .Where(static value => !string.IsNullOrWhiteSpace(value))
+                    .Select(static value => value.Trim())
+                    .Distinct(StringComparer.Ordinal)
+                    .ToArray()
+                : null,
+            RecentSearches = ambient.RecentSearches is { Count: > 0 }
+                ? ambient.RecentSearches
+                    .Where(static value => !string.IsNullOrWhiteSpace(value))
+                    .Select(static value => value.Trim())
+                    .Distinct(StringComparer.OrdinalIgnoreCase)
+                    .ToArray()
+                : null
         };
     }
 
@@ -204,7 +416,17 @@ public static class UnifiedSearchEndpoints
                 IsPrimary = action.IsPrimary
             }).ToArray(),
             Metadata = card.Metadata,
-            Sources = card.Sources.ToArray()
+            Sources = card.Sources.ToArray(),
+            Facets = card.Facets.Select(static facet => new UnifiedSearchApiFacet
+            {
+                Domain = facet.Domain,
+                Title = facet.Title,
+                Snippet = facet.Snippet,
+                Score = facet.Score,
+                Metadata = facet.Metadata
+            }).ToArray(),
+            Connections = card.Connections.ToArray(),
+            SynthesisHints = card.SynthesisHints
         }).ToArray();
 
         UnifiedSearchApiSynthesis? synthesis = null;
@@ -256,10 +478,95 @@ public static class UnifiedSearchEndpoints
                 DurationMs = response.Diagnostics.DurationMs,
                 UsedVector = response.Diagnostics.UsedVector,
                 Mode = response.Diagnostics.Mode
-            }
+            },
+            Federation = response.Diagnostics.Federation?.Select(static diag => new UnifiedSearchApiFederationDiagnostic
+            {
+                Backend = diag.Backend,
+                ResultCount = diag.ResultCount,
+                DurationMs = diag.DurationMs,
+                TimedOut = diag.TimedOut,
+                Status = diag.Status
+            }).ToArray()
         };
     }
 
+    private static bool HasSynthesisScope(HttpContext context)
+    {
+        var scopes = ResolveUserScopes(context);
+        if (scopes is null || scopes.Count == 0)
+        {
+            return false;
+        }
+
+        return scopes.Contains("search:synthesize", StringComparer.OrdinalIgnoreCase) ||
+               scopes.Contains("advisory-ai:admin", StringComparer.OrdinalIgnoreCase);
+    }
+
+    private static IReadOnlyList<EntityCard> MapSynthesisCards(IReadOnlyList<UnifiedSearchApiCard>? cards)
+    {
+        if (cards is not { Count: > 0 })
+        {
+            return [];
+        }
+
+        return cards
+            .Select(static card => new EntityCard
+            {
+                EntityKey = card.EntityKey,
+                EntityType = card.EntityType,
+                Domain = card.Domain,
+                Title = card.Title,
+                Snippet = card.Snippet,
+                Score = card.Score,
+                Severity = card.Severity,
+                Metadata = card.Metadata,
+                Sources = card.Sources,
+                Actions = card.Actions.Select(static action => new EntityCardAction(
+                    action.Label,
+                    action.ActionType,
+                    action.Route,
+                    action.Command,
+                    action.IsPrimary)).ToArray(),
+                Facets = card.Facets.Select(static facet => new EntityCardFacet
+                {
+                    Domain = facet.Domain,
+                    Title = facet.Title,
+                    Snippet = facet.Snippet,
+                    Score = facet.Score,
+                    Metadata = facet.Metadata
+                }).ToArray(),
+                Connections = card.Connections,
+                SynthesisHints = card.SynthesisHints
+            })
+            .ToArray();
+    }
+
+    private static async Task WriteSseEventAsync(
+        HttpContext context,
+        string eventName,
+        object payload,
+        CancellationToken cancellationToken)
+    {
+        var json = JsonSerializer.Serialize(payload);
+        await context.Response.WriteAsync($"event: {eventName}\n", cancellationToken).ConfigureAwait(false);
+        await context.Response.WriteAsync($"data: {json}\n\n", cancellationToken).ConfigureAwait(false);
+        await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    private static IEnumerable<string> Chunk(string content, int size)
+    {
+        if (string.IsNullOrEmpty(content) || size <= 0)
+        {
+            yield break;
+        }
+
+        for (var index = 0; index < content.Length; index += size)
+        {
+            var length = Math.Min(size, content.Length - index);
+            yield return content.Substring(index, length);
+        }
+    }
+
     private static string? ResolveTenant(HttpContext context)
     {
         foreach (var value in context.Request.Headers["X-StellaOps-Tenant"])
@@ -374,6 +681,21 @@ public sealed record UnifiedSearchApiRequest
     public bool IncludeSynthesis { get; init; } = true;
 
     public bool IncludeDebug { get; init; }
+
+    public UnifiedSearchApiAmbientContext? Ambient { get; init; }
+}
+
+public sealed record UnifiedSearchApiAmbientContext
+{
+    public string? CurrentRoute { get; init; }
+
+    public IReadOnlyList<string>? VisibleEntityKeys { get; init; }
+
+    public IReadOnlyList<string>? RecentSearches { get; init; }
+
+    public string? SessionId { get; init; }
+
+    public bool ResetSession { get; init; }
 }
 
 public sealed record UnifiedSearchApiFilter
@@ -393,6 +715,28 @@ public sealed record UnifiedSearchApiFilter
     public IReadOnlyList<string>? Tags { get; init; }
 }
 
+public sealed record UnifiedSearchSynthesizeApiRequest
+{
+    public string Q { get; init; } = string.Empty;
+
+    public IReadOnlyList<UnifiedSearchApiCard> TopCards { get; init; } = [];
+
+    public QueryPlan? Plan { get; init; }
+
+    public UnifiedSearchSynthesisPreferencesApi? Preferences { get; init; }
+}
+
+public sealed record UnifiedSearchSynthesisPreferencesApi
+{
+    public string Depth { get; init; } = "brief";
+
+    public int? MaxTokens { get; init; }
+
+    public bool IncludeActions { get; init; } = true;
+
+    public string Locale { get; init; } = "en";
+}
+
 public sealed record UnifiedSearchApiResponse
 {
     public string Query { get; init; } = string.Empty;
@@ -408,6 +752,8 @@ public sealed record UnifiedSearchApiResponse
     public IReadOnlyList<UnifiedSearchApiRefinement>? Refinements { get; init; }
 
     public UnifiedSearchApiDiagnostics Diagnostics { get; init; } = new();
+
+    public IReadOnlyList<UnifiedSearchApiFederationDiagnostic>? Federation { get; init; }
 }
 
 public sealed record UnifiedSearchApiCard
@@ -431,6 +777,26 @@ public sealed record UnifiedSearchApiCard
     public IReadOnlyDictionary<string, string>? Metadata { get; init; }
 
     public IReadOnlyList<string> Sources { get; init; } = [];
+
+    public IReadOnlyList<UnifiedSearchApiFacet> Facets { get; init; } = [];
+
+    public IReadOnlyList<string> Connections { get; init; } = [];
+
+    public IReadOnlyDictionary<string, string> SynthesisHints { get; init; } =
+        new Dictionary<string, string>(StringComparer.Ordinal);
+}
+
+public sealed record UnifiedSearchApiFacet
+{
+    public string Domain { get; init; } = "knowledge";
+
+    public string Title { get; init; } = string.Empty;
+
+    public string Snippet { get; init; } = string.Empty;
+
+    public double Score { get; init; }
+
+    public IReadOnlyDictionary<string, string>? Metadata { get; init; }
 }
 
 public sealed record UnifiedSearchApiAction
@@ -488,6 +854,19 @@ public sealed record UnifiedSearchApiDiagnostics
     public string Mode { get; init; } = "fts-only";
 }
 
+public sealed record UnifiedSearchApiFederationDiagnostic
+{
+    public string Backend { get; init; } = string.Empty;
+
+    public int ResultCount { get; init; }
+
+    public long DurationMs { get; init; }
+
+    public bool TimedOut { get; init; }
+
+    public string Status { get; init; } = "ok";
+}
+
 public sealed record UnifiedSearchRebuildApiResponse
 {
     public int DomainCount { get; init; }