stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

Harden live-backed unified search weighting and indexing

Browse Source
This commit is contained in:
master
2026-03-08 02:23:43 +02:00
parent c7b7ddf436
commit 145e67a544
26 changed files with 1585 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/AdvisoryAI/StellaOps.AdvisoryAI/KnowledgeSearch/KnowledgeIndexer.cs
View File

@@ -82,11 +82,23 @@ internal sealed class KnowledgeIndexer : IKnowledgeIndexer
private EffectiveIngestionOptions ResolveEffectiveOptions()
{
var repositoryRoot = string.IsNullOrWhiteSpace(_options.RepositoryRoot)
? Directory.GetCurrentDirectory()
: Path.IsPathRooted(_options.RepositoryRoot)
? Path.GetFullPath(_options.RepositoryRoot)
: Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), _options.RepositoryRoot));
var repositoryRootResolution = KnowledgeSearchRepositoryRootResolver.Resolve(_options);
if (!repositoryRootResolution.Validated)
{
_logger.LogWarning(
"Knowledge ingestion could not validate repository root from source {Source}; using {RepositoryRoot}. Relative corpus paths may resolve to an empty index.",
repositoryRootResolution.Source,
repositoryRootResolution.Path);
}
else if (!string.Equals(repositoryRootResolution.Source, "configured", StringComparison.Ordinal))
{
_logger.LogInformation(
"Auto-detected AdvisoryAI repository root from {Source}: {RepositoryRoot}.",
repositoryRootResolution.Source,
repositoryRootResolution.Path);
}
var repositoryRoot = repositoryRootResolution.Path;
var markdownRoots = (_options.MarkdownRoots ?? [])
.Where(static root => !string.IsNullOrWhiteSpace(root))

15
src/AdvisoryAI/StellaOps.AdvisoryAI/KnowledgeSearch/KnowledgeSearchBenchmarkDatasetGenerator.cs
View File

@@ -120,17 +120,16 @@ internal sealed class KnowledgeSearchBenchmarkDatasetGenerator : IKnowledgeSearc
private string ResolveRepositoryRoot()
{
if (string.IsNullOrWhiteSpace(_options.RepositoryRoot))
var resolution = KnowledgeSearchRepositoryRootResolver.Resolve(_options);
if (!resolution.Validated)
{
return Directory.GetCurrentDirectory();
_logger.LogWarning(
"Knowledge benchmark dataset generation could not validate repository root from source {Source}; using {RepositoryRoot}.",
resolution.Source,
resolution.Path);
}
if (Path.IsPathRooted(_options.RepositoryRoot))
{
return Path.GetFullPath(_options.RepositoryRoot);
}
return Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), _options.RepositoryRoot));
return resolution.Path;
}
private IReadOnlyList<BenchmarkTarget> LoadMarkdownTargets(string repositoryRoot, CancellationToken cancellationToken)

6
src/AdvisoryAI/StellaOps.AdvisoryAI/KnowledgeSearch/KnowledgeSearchOptions.cs
View File

@@ -55,13 +55,13 @@ public sealed class KnowledgeSearchOptions
public List<string> OpenApiRoots { get; set; } = ["src", "devops/compose"];
public string UnifiedFindingsSnapshotPath { get; set; } =
"UnifiedSearch/Snapshots/findings.snapshot.json";
"src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Snapshots/findings.snapshot.json";
public string UnifiedVexSnapshotPath { get; set; } =
"UnifiedSearch/Snapshots/vex.snapshot.json";
"src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Snapshots/vex.snapshot.json";
public string UnifiedPolicySnapshotPath { get; set; } =
"UnifiedSearch/Snapshots/policy.snapshot.json";
"src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Snapshots/policy.snapshot.json";
public bool UnifiedAutoIndexEnabled { get; set; }

195
src/AdvisoryAI/StellaOps.AdvisoryAI/KnowledgeSearch/KnowledgeSearchRepositoryRootResolver.cs Normal file
View File

@@ -0,0 +1,195 @@
namespace StellaOps.AdvisoryAI.KnowledgeSearch;
internal sealed record KnowledgeSearchRepositoryRootResolution(
string Path,
bool Validated,
string Source);
internal static class KnowledgeSearchRepositoryRootResolver
{
public static string ResolvePath(KnowledgeSearchOptions options, string configuredPath)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentException.ThrowIfNullOrWhiteSpace(configuredPath);
if (Path.IsPathRooted(configuredPath))
{
return Path.GetFullPath(configuredPath);
}
var repositoryRoot = Resolve(options);
return Path.GetFullPath(Path.Combine(repositoryRoot.Path, configuredPath));
}
public static KnowledgeSearchRepositoryRootResolution Resolve(KnowledgeSearchOptions options)
{
return Resolve(options, Directory.GetCurrentDirectory(), AppContext.BaseDirectory);
}
internal static KnowledgeSearchRepositoryRootResolution Resolve(
KnowledgeSearchOptions options,
string? currentDirectory,
string? appBaseDirectory)
{
ArgumentNullException.ThrowIfNull(options);
var configuredRoot = NormalizeConfiguredRoot(options.RepositoryRoot, currentDirectory);
var relativeMarkers = BuildRelativeMarkers(options);
foreach (var (candidate, source, allowAncestorSearch) in EnumerateCandidates(configuredRoot, currentDirectory, appBaseDirectory))
{
var resolved = FindRepositoryRoot(candidate, relativeMarkers, allowAncestorSearch);
if (resolved is null)
{
continue;
}
return new KnowledgeSearchRepositoryRootResolution(resolved, true, source);
}
if (!string.IsNullOrWhiteSpace(configuredRoot))
{
return new KnowledgeSearchRepositoryRootResolution(configuredRoot, false, "configured_fallback");
}
var fallbackPath =
NormalizeAbsolutePath(currentDirectory)
?? NormalizeAbsolutePath(appBaseDirectory)
?? Path.GetFullPath(Directory.GetCurrentDirectory());
return new KnowledgeSearchRepositoryRootResolution(fallbackPath, false, "current_directory_fallback");
}
private static IEnumerable<(string Candidate, string Source, bool AllowAncestorSearch)> EnumerateCandidates(
string? configuredRoot,
string? currentDirectory,
string? appBaseDirectory)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(configuredRoot) && seen.Add(configuredRoot))
{
yield return (configuredRoot, "configured", false);
}
var normalizedCurrentDirectory = NormalizeAbsolutePath(currentDirectory);
if (!string.IsNullOrWhiteSpace(normalizedCurrentDirectory) && seen.Add(normalizedCurrentDirectory))
{
yield return (normalizedCurrentDirectory, "current_directory", true);
}
var normalizedAppBaseDirectory = NormalizeAbsolutePath(appBaseDirectory);
if (!string.IsNullOrWhiteSpace(normalizedAppBaseDirectory) && seen.Add(normalizedAppBaseDirectory))
{
yield return (normalizedAppBaseDirectory, "app_base_directory", true);
}
}
private static string? FindRepositoryRoot(string? startPath, IReadOnlyList<string> relativeMarkers, bool allowAncestorSearch)
{
var normalizedStartPath = NormalizeAbsolutePath(startPath);
if (string.IsNullOrWhiteSpace(normalizedStartPath))
{
return null;
}
if (!allowAncestorSearch)
{
return LooksLikeRepositoryRoot(normalizedStartPath, relativeMarkers)
? normalizedStartPath
: null;
}
var current = new DirectoryInfo(normalizedStartPath);
while (current is not null)
{
if (LooksLikeRepositoryRoot(current.FullName, relativeMarkers))
{
return current.FullName;
}
current = current.Parent;
}
return null;
}
private static bool LooksLikeRepositoryRoot(string candidate, IReadOnlyList<string> relativeMarkers)
{
if (!Directory.Exists(candidate))
{
return false;
}
var hasRepoShape =
File.Exists(Path.Combine(candidate, "global.json"))
|| (Directory.Exists(Path.Combine(candidate, "src")) && Directory.Exists(Path.Combine(candidate, "docs")));
if (!hasRepoShape)
{
return false;
}
foreach (var marker in relativeMarkers)
{
var markerPath = Path.Combine(candidate, marker);
if (Directory.Exists(markerPath) || File.Exists(markerPath))
{
return true;
}
}
return false;
}
private static string? NormalizeConfiguredRoot(string? configuredRoot, string? currentDirectory)
{
if (string.IsNullOrWhiteSpace(configuredRoot))
{
return null;
}
if (Path.IsPathRooted(configuredRoot))
{
return Path.GetFullPath(configuredRoot);
}
var baseDirectory = NormalizeAbsolutePath(currentDirectory) ?? Path.GetFullPath(Directory.GetCurrentDirectory());
return Path.GetFullPath(Path.Combine(baseDirectory, configuredRoot));
}
private static List<string> BuildRelativeMarkers(KnowledgeSearchOptions options)
{
var markers = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
AddRelativeMarker(markers, options.MarkdownAllowListPath);
AddRelativeMarker(markers, options.DoctorSeedPath);
AddRelativeMarker(markers, options.DoctorControlsPath);
AddRelativeMarker(markers, options.OpenApiAggregatePath);
AddRelativeMarker(markers, "docs");
AddRelativeMarker(markers, "src");
AddRelativeMarker(markers, "devops");
return markers.ToList();
}
private static void AddRelativeMarker(ISet<string> markers, string? value)
{
if (string.IsNullOrWhiteSpace(value) || Path.IsPathRooted(value))
{
return;
}
markers.Add(value.Trim());
}
private static string? NormalizeAbsolutePath(string? path)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
return Path.GetFullPath(path);
}
}

2
src/AdvisoryAI/StellaOps.AdvisoryAI/TASKS.md
View File

@@ -17,7 +17,7 @@ Source of truth: `docs/implplan/SPRINT_20260113_005_ADVISORYAI_controlled_conver
| AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. |
| SPRINT_20260307_019-AI-ZL | DONE | Unified search now applies implicit current-scope weighting, emits additive `overflow`/`coverage`, blends close top answers, and evaluates suggestion viability without requiring telemetry. |
| SPRINT_20260307_033-AI-ZL | DONE | Unified search now derives answer blending from query/context, exposes grounded-only suggestion viability with corpus-readiness states, and keeps analytics/feedback telemetry fully optional. |
| SPRINT_20260307_037-AI-SF | TODO | Final search-first correction pass: stronger in-scope weighting, stricter blended-answer thresholds, grounded-only supported-route suggestion viability, and telemetry-independent correctness. |
| SPRINT_20260307_037-AI-SF | DONE | Final search-first correction pass: stronger in-scope weighting, stricter blended-answer thresholds, grounded-only supported-route suggestion viability, and telemetry-independent correctness. |
| SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. |
| AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. |
| AUDIT-0017-T | DONE | Test coverage audit for StellaOps.AdvisoryAI. |

81
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Adapters/FindingIngestionAdapter.cs
View File

@@ -68,9 +68,18 @@ internal sealed class FindingIngestionAdapter : ISearchIngestionAdapter
var tenant = ReadString(entry, "tenant") ?? "global";
var tags = ReadStringArray(entry, "tags", ["finding", "vulnerability", severity]);
var body = string.IsNullOrWhiteSpace(description)
? $"{title}\nSeverity: {severity}"
: $"{title}\n{description}\nSeverity: {severity}";
var body = BuildBody(
findingId,
cveId,
title,
description,
severity,
service,
reachability: null,
environment: null,
product: null,
policyBadge: null,
tags);
var chunkId = KnowledgeSearchText.StableId("chunk", "finding", findingId, cveId);
var docId = KnowledgeSearchText.StableId("doc", "finding", findingId);
var embedding = _vectorEncoder.Encode(body);
@@ -116,15 +125,71 @@ internal sealed class FindingIngestionAdapter : ISearchIngestionAdapter
}));
}
private string ResolvePath(string configuredPath)
private static string BuildBody(
string findingId,
string cveId,
string title,
string description,
string severity,
string service,
string? reachability,
string? environment,
string? product,
string? policyBadge,
IReadOnlyList<string> tags)
{
if (Path.IsPathRooted(configuredPath))
var bodyParts = new List<string>
{
return configuredPath;
$"Finding: {title}",
$"Finding ID: {findingId}",
$"CVE: {cveId}",
"Finding type: vulnerability finding",
$"Severity: {severity}",
$"Reachable: {NormalizeTextOrDefault(reachability, "unknown")}",
"Status: open unresolved"
};
if (!string.IsNullOrWhiteSpace(product))
{
bodyParts.Add($"Product: {product}");
}
var root = string.IsNullOrWhiteSpace(_options.RepositoryRoot) ? "." : _options.RepositoryRoot;
return Path.GetFullPath(Path.Combine(root, configuredPath));
if (!string.IsNullOrWhiteSpace(environment))
{
bodyParts.Add($"Environment: {environment}");
}
if (!string.IsNullOrWhiteSpace(policyBadge))
{
bodyParts.Add($"Policy gate: {policyBadge}");
}
if (!string.IsNullOrWhiteSpace(service))
{
bodyParts.Add($"Service: {service}");
}
if (tags.Count > 0)
{
bodyParts.Add($"Tags: {string.Join(", ", tags)}");
}
if (!string.IsNullOrWhiteSpace(description))
{
bodyParts.Add(description);
}
return string.Join("\n", bodyParts);
}
private static string NormalizeTextOrDefault(string? value, string fallback)
{
return string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
}
private string ResolvePath(string configuredPath)
{
return KnowledgeSearchRepositoryRootResolver.ResolvePath(_options, configuredPath);
}
private static string? ReadString(JsonElement obj, string propertyName)

126
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Adapters/FindingsSearchAdapter.cs
View File

@@ -181,23 +181,19 @@ internal sealed class FindingsSearchAdapter : ISearchIngestionAdapter
? $"{cveId} [{severity}]"
: $"{cveId} - {component} [{severity}]";
var bodyParts = new List<string> { title };
if (!string.IsNullOrWhiteSpace(description))
{
bodyParts.Add(description);
}
if (!string.IsNullOrWhiteSpace(reachability))
{
bodyParts.Add($"Reachability: {reachability}");
}
if (!string.IsNullOrWhiteSpace(environment))
{
bodyParts.Add($"Environment: {environment}");
}
bodyParts.Add($"Severity: {severity}");
var body = string.Join("\n", bodyParts);
var body = BuildBody(
findingId,
cveId,
title,
description,
severity,
"scanner",
reachability,
environment,
product,
policyBadge,
tags,
status: ReadString(entry, "status") ?? ReadString(entry, "Status"));
// Scope ids by tenant to prevent cross-tenant overwrite collisions
// when different tenants have identical finding ids/cve pairs.
var chunkId = KnowledgeSearchText.StableId("chunk", "finding", tenantIdentity, findingId, cveId);
@@ -279,9 +275,19 @@ internal sealed class FindingsSearchAdapter : ISearchIngestionAdapter
var tenantIdentity = NormalizeTenantForIdentity(tenant);
var tags = ReadStringArray(entry, "tags", ["finding", "vulnerability", severity]);
var body = string.IsNullOrWhiteSpace(description)
? $"{title}\nSeverity: {severity}"
: $"{title}\n{description}\nSeverity: {severity}";
var body = BuildBody(
findingId,
cveId,
title,
description,
severity,
service,
reachability: null,
environment: null,
product: service,
policyBadge: null,
tags,
status: null);
// Scope ids by tenant to prevent cross-tenant overwrite collisions
// when different tenants have identical finding ids/cve pairs.
var chunkId = KnowledgeSearchText.StableId("chunk", "finding", tenantIdentity, findingId, cveId);
@@ -331,15 +337,85 @@ internal sealed class FindingsSearchAdapter : ISearchIngestionAdapter
}));
}
private string ResolvePath(string configuredPath)
private static string BuildBody(
string findingId,
string cveId,
string title,
string description,
string severity,
string service,
string? reachability,
string? environment,
string? product,
string? policyBadge,
IReadOnlyList<string> tags,
string? status)
{
if (Path.IsPathRooted(configuredPath))
var bodyParts = new List<string>
{
return configuredPath;
$"Finding: {title}",
$"Finding ID: {findingId}",
$"CVE: {cveId}",
"Finding type: vulnerability finding",
$"Severity: {severity}",
$"Reachable: {NormalizeTextOrDefault(reachability, "unknown")}",
$"Status: {ResolveFindingStatus(status)}"
};
if (!string.IsNullOrWhiteSpace(product))
{
bodyParts.Add($"Product: {product}");
}
var root = string.IsNullOrWhiteSpace(_options.RepositoryRoot) ? "." : _options.RepositoryRoot;
return Path.GetFullPath(Path.Combine(root, configuredPath));
if (!string.IsNullOrWhiteSpace(environment))
{
bodyParts.Add($"Environment: {environment}");
}
if (!string.IsNullOrWhiteSpace(policyBadge))
{
bodyParts.Add($"Policy gate: {policyBadge}");
}
if (!string.IsNullOrWhiteSpace(service))
{
bodyParts.Add($"Service: {service}");
}
if (tags.Count > 0)
{
bodyParts.Add($"Tags: {string.Join(", ", tags)}");
}
if (!string.IsNullOrWhiteSpace(description))
{
bodyParts.Add(description);
}
return string.Join("\n", bodyParts);
}
private static string ResolveFindingStatus(string? status)
{
if (string.IsNullOrWhiteSpace(status))
{
return "open unresolved";
}
var normalized = status.Trim().Replace('_', ' ');
return normalized.Contains("open", StringComparison.OrdinalIgnoreCase)
? normalized
: $"{normalized} unresolved";
}
private static string NormalizeTextOrDefault(string? value, string fallback)
{
return string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
}
private string ResolvePath(string configuredPath)
{
return KnowledgeSearchRepositoryRootResolver.ResolvePath(_options, configuredPath);
}
private static string? ReadString(JsonElement obj, string propertyName)

92
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Adapters/PolicyRuleIngestionAdapter.cs
View File

@@ -67,9 +67,15 @@ internal sealed class PolicyRuleIngestionAdapter : ISearchIngestionAdapter
var tenant = ReadString(entry, "tenant") ?? "global";
var tags = ReadStringArray(entry, "tags", ["policy", "rule"]);
var body = string.IsNullOrWhiteSpace(decision)
? $"{title}\nRule: {ruleId}\n{description}"
: $"{title}\nRule: {ruleId}\nDecision: {decision}\n{description}";
var body = BuildBody(
ruleId,
title,
description,
decision,
service,
scope: null,
environment: null,
tags);
var chunkId = KnowledgeSearchText.StableId("chunk", "policy_rule", ruleId);
var docId = KnowledgeSearchText.StableId("doc", "policy_rule", ruleId);
var embedding = _vectorEncoder.Encode(body);
@@ -113,15 +119,85 @@ internal sealed class PolicyRuleIngestionAdapter : ISearchIngestionAdapter
}));
}
private string ResolvePath(string configuredPath)
private static string BuildBody(
string ruleId,
string title,
string description,
string? decision,
string service,
string? scope,
string? environment,
IReadOnlyList<string> tags)
{
if (Path.IsPathRooted(configuredPath))
var normalizedDecision = NormalizeTextOrDefault(decision, "unknown");
var bodyParts = new List<string>
{
return configuredPath;
$"Policy rule: {title}",
$"Rule ID: {ruleId}",
$"Gate decision: {normalizedDecision}",
$"Gate state: {ResolveGateState(normalizedDecision)}",
"Policy domain: policy rule gate"
};
if (!string.IsNullOrWhiteSpace(scope))
{
bodyParts.Add($"Scope: {scope}");
}
var root = string.IsNullOrWhiteSpace(_options.RepositoryRoot) ? "." : _options.RepositoryRoot;
return Path.GetFullPath(Path.Combine(root, configuredPath));
if (!string.IsNullOrWhiteSpace(environment))
{
bodyParts.Add($"Environment: {environment}");
}
if (!string.IsNullOrWhiteSpace(service))
{
bodyParts.Add($"Service: {service}");
}
if (tags.Count > 0)
{
bodyParts.Add($"Tags: {string.Join(", ", tags)}");
}
if (!string.IsNullOrWhiteSpace(description))
{
bodyParts.Add(description);
}
return string.Join("\n", bodyParts);
}
private static string ResolveGateState(string decision)
{
if (decision.Contains("deny", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("block", StringComparison.OrdinalIgnoreCase))
{
return "failing blocking";
}
if (decision.Contains("warn", StringComparison.OrdinalIgnoreCase))
{
return "warning advisory";
}
if (decision.Contains("pass", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("allow", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("require", StringComparison.OrdinalIgnoreCase))
{
return "passing";
}
return "unknown";
}
private static string NormalizeTextOrDefault(string? value, string fallback)
{
return string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
}
private string ResolvePath(string configuredPath)
{
return KnowledgeSearchRepositoryRootResolver.ResolvePath(_options, configuredPath);
}
private static string? ReadString(JsonElement obj, string propertyName)

135
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Adapters/PolicySearchAdapter.cs
View File

@@ -191,21 +191,17 @@ internal sealed class PolicySearchAdapter : ISearchIngestionAdapter
? $"{ruleId} [{enforcement}]"
: $"{ruleId} - {bomRef} [{enforcement}]";
var bodyParts = new List<string> { title, $"Rule: {ruleId}", $"Enforcement: {enforcement}" };
if (!string.IsNullOrWhiteSpace(description))
{
bodyParts.Add(description);
}
if (!string.IsNullOrWhiteSpace(bomRef))
{
bodyParts.Add($"Scope: {bomRef}");
}
if (!string.IsNullOrWhiteSpace(verdictHash))
{
bodyParts.Add($"Verdict: {verdictHash}");
}
var body = string.Join("\n", bodyParts);
var body = BuildBody(
ruleId,
title,
description,
enforcement,
scope,
environment,
verdictHash,
ciContext,
actor,
tags);
// Scope ids by tenant to prevent cross-tenant overwrite collisions
// when rule ids are reused in different tenants.
var chunkId = KnowledgeSearchText.StableId("chunk", "policy_rule", tenantIdentity, ruleId);
@@ -289,9 +285,17 @@ internal sealed class PolicySearchAdapter : ISearchIngestionAdapter
var tenantIdentity = NormalizeTenantForIdentity(tenant);
var tags = ReadStringArray(entry, "tags", ["policy", "rule"]);
var body = string.IsNullOrWhiteSpace(decision)
? $"{title}\nRule: {ruleId}\n{description}"
: $"{title}\nRule: {ruleId}\nDecision: {decision}\n{description}";
var body = BuildBody(
ruleId,
title,
description,
decision,
scope: string.Empty,
environment: string.Empty,
verdictHash: string.Empty,
ciContext: string.Empty,
actor: string.Empty,
tags);
// Scope ids by tenant to prevent cross-tenant overwrite collisions
// when rule ids are reused in different tenants.
var chunkId = KnowledgeSearchText.StableId("chunk", "policy_rule", tenantIdentity, ruleId);
@@ -339,15 +343,100 @@ internal sealed class PolicySearchAdapter : ISearchIngestionAdapter
}));
}
private string ResolvePath(string configuredPath)
private static string BuildBody(
string ruleId,
string title,
string description,
string? decision,
string? scope,
string? environment,
string? verdictHash,
string? ciContext,
string? actor,
IReadOnlyList<string> tags)
{
if (Path.IsPathRooted(configuredPath))
var normalizedDecision = NormalizeTextOrDefault(decision, "unknown");
var bodyParts = new List<string>
{
return configuredPath;
$"Policy rule: {title}",
$"Rule ID: {ruleId}",
$"Gate decision: {normalizedDecision}",
$"Gate state: {ResolveGateState(normalizedDecision)}",
"Policy domain: policy rule gate"
};
if (!string.IsNullOrWhiteSpace(scope))
{
bodyParts.Add($"Scope: {scope}");
}
var root = string.IsNullOrWhiteSpace(_options.RepositoryRoot) ? "." : _options.RepositoryRoot;
return Path.GetFullPath(Path.Combine(root, configuredPath));
if (!string.IsNullOrWhiteSpace(environment))
{
bodyParts.Add($"Environment: {environment}");
}
if (!string.IsNullOrWhiteSpace(verdictHash))
{
bodyParts.Add($"Verdict: {verdictHash}");
}
if (!string.IsNullOrWhiteSpace(ciContext))
{
bodyParts.Add($"CI context: {ciContext}");
}
if (!string.IsNullOrWhiteSpace(actor))
{
bodyParts.Add($"Actor: {actor}");
}
if (tags.Count > 0)
{
bodyParts.Add($"Tags: {string.Join(", ", tags)}");
}
if (!string.IsNullOrWhiteSpace(description))
{
bodyParts.Add(description);
}
return string.Join("\n", bodyParts);
}
private static string ResolveGateState(string decision)
{
if (decision.Contains("deny", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("block", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("mandatory", StringComparison.OrdinalIgnoreCase))
{
return "failing blocking";
}
if (decision.Contains("warn", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("advisory", StringComparison.OrdinalIgnoreCase))
{
return "warning advisory";
}
if (decision.Contains("pass", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("allow", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("informational", StringComparison.OrdinalIgnoreCase) ||
decision.Contains("require", StringComparison.OrdinalIgnoreCase))
{
return "passing";
}
return "unknown";
}
private static string NormalizeTextOrDefault(string? value, string fallback)
{
return string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
}
private string ResolvePath(string configuredPath)
{
return KnowledgeSearchRepositoryRootResolver.ResolvePath(_options, configuredPath);
}
private static string? ReadString(JsonElement obj, string propertyName)

92
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Adapters/VexSearchAdapter.cs
View File

@@ -187,21 +187,16 @@ internal sealed class VexSearchAdapter : ISearchIngestionAdapter
? $"VEX: {cveId} ({status})"
: $"VEX: {cveId} - {product} ({status})";
var bodyParts = new List<string> { title, $"Status: {status}" };
if (!string.IsNullOrWhiteSpace(justification))
{
bodyParts.Add($"Justification: {justification}");
}
if (!string.IsNullOrWhiteSpace(advisoryTitle))
{
bodyParts.Add($"Advisory: {advisoryTitle}");
}
if (!string.IsNullOrWhiteSpace(severity))
{
bodyParts.Add($"Severity: {severity}");
}
var body = string.Join("\n", bodyParts);
var body = BuildBody(
statementId,
cveId,
title,
status,
justification,
product,
advisoryTitle,
severity,
tags);
// Scope ids by tenant to prevent cross-tenant overwrite collisions
// when statement ids are reused in different tenants.
var chunkId = KnowledgeSearchText.StableId("chunk", "vex_statement", tenantIdentity, statementId);
@@ -283,9 +278,16 @@ internal sealed class VexSearchAdapter : ISearchIngestionAdapter
var tags = ReadStringArray(entry, "tags", ["vex", "statement", status]);
var title = $"VEX: {cveId} ({status})";
var body = string.IsNullOrWhiteSpace(justification)
? $"{title}\nStatus: {status}"
: $"{title}\nStatus: {status}\nJustification: {justification}";
var body = BuildBody(
statementId,
cveId,
title,
status,
justification,
product: null,
advisoryTitle: null,
severity: null,
tags);
// Scope ids by tenant to prevent cross-tenant overwrite collisions
// when statement ids are reused in different tenants.
var chunkId = KnowledgeSearchText.StableId("chunk", "vex_statement", tenantIdentity, statementId);
@@ -333,15 +335,59 @@ internal sealed class VexSearchAdapter : ISearchIngestionAdapter
}));
}
private string ResolvePath(string configuredPath)
private static string BuildBody(
string statementId,
string cveId,
string title,
string status,
string justification,
string? product,
string? advisoryTitle,
string? severity,
IReadOnlyList<string> tags)
{
if (Path.IsPathRooted(configuredPath))
var statusLabel = status.Replace('_', ' ');
var bodyParts = new List<string>
{
return configuredPath;
$"VEX statement: {title}",
$"Statement ID: {statementId}",
$"CVE: {cveId}",
$"Disposition: {statusLabel}",
$"Marked as: {statusLabel}",
"Conflict evidence: none recorded"
};
if (!string.IsNullOrWhiteSpace(product))
{
bodyParts.Add($"Covered component: {product}");
}
var root = string.IsNullOrWhiteSpace(_options.RepositoryRoot) ? "." : _options.RepositoryRoot;
return Path.GetFullPath(Path.Combine(root, configuredPath));
if (!string.IsNullOrWhiteSpace(advisoryTitle))
{
bodyParts.Add($"Advisory: {advisoryTitle}");
}
if (!string.IsNullOrWhiteSpace(severity))
{
bodyParts.Add($"Severity: {severity}");
}
if (tags.Count > 0)
{
bodyParts.Add($"Tags: {string.Join(", ", tags.Select(static tag => tag.Replace('_', ' ')))}");
}
if (!string.IsNullOrWhiteSpace(justification))
{
bodyParts.Add($"Justification: {justification}");
}
return string.Join("\n", bodyParts);
}
private string ResolvePath(string configuredPath)
{
return KnowledgeSearchRepositoryRootResolver.ResolvePath(_options, configuredPath);
}
private static string? ReadString(JsonElement obj, string propertyName)

53
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Adapters/VexStatementIngestionAdapter.cs
View File

@@ -68,9 +68,14 @@ internal sealed class VexStatementIngestionAdapter : ISearchIngestionAdapter
var tags = ReadStringArray(entry, "tags", ["vex", "statement", status]);
var title = $"VEX: {cveId} ({status})";
var body = string.IsNullOrWhiteSpace(justification)
? $"{title}\nStatus: {status}"
: $"{title}\nStatus: {status}\nJustification: {justification}";
var body = BuildBody(
statementId,
cveId,
title,
status,
justification,
product: null,
tags);
var chunkId = KnowledgeSearchText.StableId("chunk", "vex_statement", statementId);
var docId = KnowledgeSearchText.StableId("doc", "vex_statement", cveId);
var embedding = _vectorEncoder.Encode(body);
@@ -116,15 +121,47 @@ internal sealed class VexStatementIngestionAdapter : ISearchIngestionAdapter
}));
}
private string ResolvePath(string configuredPath)
private static string BuildBody(
string statementId,
string cveId,
string title,
string status,
string justification,
string? product,
IReadOnlyList<string> tags)
{
if (Path.IsPathRooted(configuredPath))
var statusLabel = status.Replace('_', ' ');
var bodyParts = new List<string>
{
return configuredPath;
$"VEX statement: {title}",
$"Statement ID: {statementId}",
$"CVE: {cveId}",
$"Disposition: {statusLabel}",
$"Marked as: {statusLabel}",
"Conflict evidence: none recorded"
};
if (!string.IsNullOrWhiteSpace(product))
{
bodyParts.Add($"Covered component: {product}");
}
var root = string.IsNullOrWhiteSpace(_options.RepositoryRoot) ? "." : _options.RepositoryRoot;
return Path.GetFullPath(Path.Combine(root, configuredPath));
if (tags.Count > 0)
{
bodyParts.Add($"Tags: {string.Join(", ", tags.Select(static tag => tag.Replace('_', ' ')))}");
}
if (!string.IsNullOrWhiteSpace(justification))
{
bodyParts.Add($"Justification: {justification}");
}
return string.Join("\n", bodyParts);
}
private string ResolvePath(string configuredPath)
{
return KnowledgeSearchRepositoryRootResolver.ResolvePath(_options, configuredPath);
}
private static string? ReadString(JsonElement obj, string propertyName)

21
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsService.cs
View File

@@ -349,13 +349,20 @@ internal sealed class SearchAnalyticsService
await conn.OpenAsync(ct).ConfigureAwait(false);
await using var cmd = new NpgsqlCommand(@"
SELECT DISTINCT query
FROM advisoryai.search_history
WHERE tenant_id = @tenant_id
AND result_count > 0
AND lower(query) <> lower(@query)
AND similarity(query, @query) > 0.2
ORDER BY similarity(query, @query) DESC
SELECT candidate_query
FROM (
SELECT
lower(query) AS normalized_query,
MIN(query) AS candidate_query,
MAX(similarity(query, @query)) AS similarity_score
FROM advisoryai.search_history
WHERE tenant_id = @tenant_id
AND result_count > 0
AND lower(query) <> lower(@query)
AND similarity(query, @query) > 0.2
GROUP BY lower(query)
) AS ranked
ORDER BY similarity_score DESC, candidate_query ASC
LIMIT @limit", conn);
cmd.CommandTimeout = 5;

4
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Context/AmbientContextProcessor.cs
View File

@@ -2,8 +2,8 @@ namespace StellaOps.AdvisoryAI.UnifiedSearch.Context;
internal sealed class AmbientContextProcessor
{
private const double CurrentRouteBoost = 0.35d;
private const double LastActionDomainBoost = 0.15d;
private const double CurrentRouteBoost = 0.85d;
private const double LastActionDomainBoost = 0.20d;
private const double VisibleEntityBoost = 0.20d;
private const double LastActionEntityBoost = 0.25d;
private static readonly (string Prefix, string Domain)[] RouteDomainMappings =

241
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchIndexer.cs
View File

@@ -5,6 +5,7 @@ using NpgsqlTypes;
using StellaOps.AdvisoryAI.KnowledgeSearch;
using System.Text.Json;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
namespace StellaOps.AdvisoryAI.UnifiedSearch;
@@ -254,6 +255,7 @@ internal sealed class UnifiedSearchIndexer : IUnifiedSearchIndexer
{
await using var dataSource = new NpgsqlDataSourceBuilder(_options.ConnectionString).Build();
await using var connection = await dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
var hasEmbeddingVectorColumn = await HasEmbeddingVectorColumnAsync(connection, cancellationToken).ConfigureAwait(false);
// Ensure parent documents exist for each unique DocId
var uniqueDocIds = chunks.Select(static c => c.DocId).Distinct(StringComparer.Ordinal).ToArray();
@@ -263,57 +265,163 @@ internal sealed class UnifiedSearchIndexer : IUnifiedSearchIndexer
await EnsureDocumentExistsAsync(connection, docId, chunk, cancellationToken).ConfigureAwait(false);
}
const string sql = """
INSERT INTO advisoryai.kb_chunk
(
chunk_id, doc_id, kind, anchor, section_path,
span_start, span_end, title, body, body_tsv,
embedding, metadata, domain, entity_key, entity_type, freshness,
indexed_at
)
VALUES
(
@chunk_id, @doc_id, @kind, @anchor, @section_path,
@span_start, @span_end, @title, @body,
setweight(to_tsvector('simple', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('simple', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('simple', coalesce(@body, '')), 'D'),
@embedding, @metadata::jsonb, @domain, @entity_key, @entity_type, @freshness,
NOW()
)
ON CONFLICT (chunk_id) DO UPDATE SET
doc_id = EXCLUDED.doc_id,
kind = EXCLUDED.kind,
anchor = EXCLUDED.anchor,
section_path = EXCLUDED.section_path,
span_start = EXCLUDED.span_start,
span_end = EXCLUDED.span_end,
title = EXCLUDED.title,
body = EXCLUDED.body,
body_tsv = EXCLUDED.body_tsv,
embedding = EXCLUDED.embedding,
metadata = EXCLUDED.metadata,
domain = EXCLUDED.domain,
entity_key = EXCLUDED.entity_key,
entity_type = EXCLUDED.entity_type,
freshness = EXCLUDED.freshness,
indexed_at = NOW()
WHERE advisoryai.kb_chunk.doc_id IS DISTINCT FROM EXCLUDED.doc_id
OR advisoryai.kb_chunk.kind IS DISTINCT FROM EXCLUDED.kind
OR advisoryai.kb_chunk.anchor IS DISTINCT FROM EXCLUDED.anchor
OR advisoryai.kb_chunk.section_path IS DISTINCT FROM EXCLUDED.section_path
OR advisoryai.kb_chunk.span_start IS DISTINCT FROM EXCLUDED.span_start
OR advisoryai.kb_chunk.span_end IS DISTINCT FROM EXCLUDED.span_end
OR advisoryai.kb_chunk.title IS DISTINCT FROM EXCLUDED.title
OR advisoryai.kb_chunk.body IS DISTINCT FROM EXCLUDED.body
OR advisoryai.kb_chunk.body_tsv IS DISTINCT FROM EXCLUDED.body_tsv
OR advisoryai.kb_chunk.embedding IS DISTINCT FROM EXCLUDED.embedding
OR advisoryai.kb_chunk.metadata IS DISTINCT FROM EXCLUDED.metadata
OR advisoryai.kb_chunk.domain IS DISTINCT FROM EXCLUDED.domain
OR advisoryai.kb_chunk.entity_key IS DISTINCT FROM EXCLUDED.entity_key
OR advisoryai.kb_chunk.entity_type IS DISTINCT FROM EXCLUDED.entity_type
OR advisoryai.kb_chunk.freshness IS DISTINCT FROM EXCLUDED.freshness;
""";
var sql = hasEmbeddingVectorColumn
? """
INSERT INTO advisoryai.kb_chunk
(
chunk_id, doc_id, kind, anchor, section_path,
span_start, span_end, title, body, body_tsv,
body_tsv_en, body_tsv_de, body_tsv_fr, body_tsv_es, body_tsv_ru,
embedding, embedding_vec, metadata, domain, entity_key, entity_type, freshness,
indexed_at
)
VALUES
(
@chunk_id, @doc_id, @kind, @anchor, @section_path,
@span_start, @span_end, @title, @body,
setweight(to_tsvector('simple', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('simple', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('simple', coalesce(@body, '')), 'D'),
setweight(to_tsvector('english', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('english', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('english', coalesce(@body, '')), 'D'),
setweight(to_tsvector('german', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('german', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('german', coalesce(@body, '')), 'D'),
setweight(to_tsvector('french', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('french', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('french', coalesce(@body, '')), 'D'),
setweight(to_tsvector('spanish', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('spanish', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('spanish', coalesce(@body, '')), 'D'),
setweight(to_tsvector('russian', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('russian', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('russian', coalesce(@body, '')), 'D'),
@embedding, CAST(@embedding_vector AS vector), @metadata::jsonb, @domain, @entity_key, @entity_type, @freshness,
NOW()
)
ON CONFLICT (chunk_id) DO UPDATE SET
doc_id = EXCLUDED.doc_id,
kind = EXCLUDED.kind,
anchor = EXCLUDED.anchor,
section_path = EXCLUDED.section_path,
span_start = EXCLUDED.span_start,
span_end = EXCLUDED.span_end,
title = EXCLUDED.title,
body = EXCLUDED.body,
body_tsv = EXCLUDED.body_tsv,
body_tsv_en = EXCLUDED.body_tsv_en,
body_tsv_de = EXCLUDED.body_tsv_de,
body_tsv_fr = EXCLUDED.body_tsv_fr,
body_tsv_es = EXCLUDED.body_tsv_es,
body_tsv_ru = EXCLUDED.body_tsv_ru,
embedding = EXCLUDED.embedding,
embedding_vec = EXCLUDED.embedding_vec,
metadata = EXCLUDED.metadata,
domain = EXCLUDED.domain,
entity_key = EXCLUDED.entity_key,
entity_type = EXCLUDED.entity_type,
freshness = EXCLUDED.freshness,
indexed_at = NOW()
WHERE advisoryai.kb_chunk.doc_id IS DISTINCT FROM EXCLUDED.doc_id
OR advisoryai.kb_chunk.kind IS DISTINCT FROM EXCLUDED.kind
OR advisoryai.kb_chunk.anchor IS DISTINCT FROM EXCLUDED.anchor
OR advisoryai.kb_chunk.section_path IS DISTINCT FROM EXCLUDED.section_path
OR advisoryai.kb_chunk.span_start IS DISTINCT FROM EXCLUDED.span_start
OR advisoryai.kb_chunk.span_end IS DISTINCT FROM EXCLUDED.span_end
OR advisoryai.kb_chunk.title IS DISTINCT FROM EXCLUDED.title
OR advisoryai.kb_chunk.body IS DISTINCT FROM EXCLUDED.body
OR advisoryai.kb_chunk.body_tsv IS DISTINCT FROM EXCLUDED.body_tsv
OR advisoryai.kb_chunk.body_tsv_en IS DISTINCT FROM EXCLUDED.body_tsv_en
OR advisoryai.kb_chunk.body_tsv_de IS DISTINCT FROM EXCLUDED.body_tsv_de
OR advisoryai.kb_chunk.body_tsv_fr IS DISTINCT FROM EXCLUDED.body_tsv_fr
OR advisoryai.kb_chunk.body_tsv_es IS DISTINCT FROM EXCLUDED.body_tsv_es
OR advisoryai.kb_chunk.body_tsv_ru IS DISTINCT FROM EXCLUDED.body_tsv_ru
OR advisoryai.kb_chunk.embedding IS DISTINCT FROM EXCLUDED.embedding
OR advisoryai.kb_chunk.embedding_vec IS DISTINCT FROM EXCLUDED.embedding_vec
OR advisoryai.kb_chunk.metadata IS DISTINCT FROM EXCLUDED.metadata
OR advisoryai.kb_chunk.domain IS DISTINCT FROM EXCLUDED.domain
OR advisoryai.kb_chunk.entity_key IS DISTINCT FROM EXCLUDED.entity_key
OR advisoryai.kb_chunk.entity_type IS DISTINCT FROM EXCLUDED.entity_type
OR advisoryai.kb_chunk.freshness IS DISTINCT FROM EXCLUDED.freshness;
"""
: """
INSERT INTO advisoryai.kb_chunk
(
chunk_id, doc_id, kind, anchor, section_path,
span_start, span_end, title, body, body_tsv,
body_tsv_en, body_tsv_de, body_tsv_fr, body_tsv_es, body_tsv_ru,
embedding, metadata, domain, entity_key, entity_type, freshness,
indexed_at
)
VALUES
(
@chunk_id, @doc_id, @kind, @anchor, @section_path,
@span_start, @span_end, @title, @body,
setweight(to_tsvector('simple', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('simple', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('simple', coalesce(@body, '')), 'D'),
setweight(to_tsvector('english', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('english', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('english', coalesce(@body, '')), 'D'),
setweight(to_tsvector('german', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('german', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('german', coalesce(@body, '')), 'D'),
setweight(to_tsvector('french', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('french', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('french', coalesce(@body, '')), 'D'),
setweight(to_tsvector('spanish', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('spanish', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('spanish', coalesce(@body, '')), 'D'),
setweight(to_tsvector('russian', coalesce(@title, '')), 'A') ||
setweight(to_tsvector('russian', coalesce(@section_path, '')), 'B') ||
setweight(to_tsvector('russian', coalesce(@body, '')), 'D'),
@embedding, @metadata::jsonb, @domain, @entity_key, @entity_type, @freshness,
NOW()
)
ON CONFLICT (chunk_id) DO UPDATE SET
doc_id = EXCLUDED.doc_id,
kind = EXCLUDED.kind,
anchor = EXCLUDED.anchor,
section_path = EXCLUDED.section_path,
span_start = EXCLUDED.span_start,
span_end = EXCLUDED.span_end,
title = EXCLUDED.title,
body = EXCLUDED.body,
body_tsv = EXCLUDED.body_tsv,
body_tsv_en = EXCLUDED.body_tsv_en,
body_tsv_de = EXCLUDED.body_tsv_de,
body_tsv_fr = EXCLUDED.body_tsv_fr,
body_tsv_es = EXCLUDED.body_tsv_es,
body_tsv_ru = EXCLUDED.body_tsv_ru,
embedding = EXCLUDED.embedding,
metadata = EXCLUDED.metadata,
domain = EXCLUDED.domain,
entity_key = EXCLUDED.entity_key,
entity_type = EXCLUDED.entity_type,
freshness = EXCLUDED.freshness,
indexed_at = NOW()
WHERE advisoryai.kb_chunk.doc_id IS DISTINCT FROM EXCLUDED.doc_id
OR advisoryai.kb_chunk.kind IS DISTINCT FROM EXCLUDED.kind
OR advisoryai.kb_chunk.anchor IS DISTINCT FROM EXCLUDED.anchor
OR advisoryai.kb_chunk.section_path IS DISTINCT FROM EXCLUDED.section_path
OR advisoryai.kb_chunk.span_start IS DISTINCT FROM EXCLUDED.span_start
OR advisoryai.kb_chunk.span_end IS DISTINCT FROM EXCLUDED.span_end
OR advisoryai.kb_chunk.title IS DISTINCT FROM EXCLUDED.title
OR advisoryai.kb_chunk.body IS DISTINCT FROM EXCLUDED.body
OR advisoryai.kb_chunk.body_tsv IS DISTINCT FROM EXCLUDED.body_tsv
OR advisoryai.kb_chunk.body_tsv_en IS DISTINCT FROM EXCLUDED.body_tsv_en
OR advisoryai.kb_chunk.body_tsv_de IS DISTINCT FROM EXCLUDED.body_tsv_de
OR advisoryai.kb_chunk.body_tsv_fr IS DISTINCT FROM EXCLUDED.body_tsv_fr
OR advisoryai.kb_chunk.body_tsv_es IS DISTINCT FROM EXCLUDED.body_tsv_es
OR advisoryai.kb_chunk.body_tsv_ru IS DISTINCT FROM EXCLUDED.body_tsv_ru
OR advisoryai.kb_chunk.embedding IS DISTINCT FROM EXCLUDED.embedding
OR advisoryai.kb_chunk.metadata IS DISTINCT FROM EXCLUDED.metadata
OR advisoryai.kb_chunk.domain IS DISTINCT FROM EXCLUDED.domain
OR advisoryai.kb_chunk.entity_key IS DISTINCT FROM EXCLUDED.entity_key
OR advisoryai.kb_chunk.entity_type IS DISTINCT FROM EXCLUDED.entity_type
OR advisoryai.kb_chunk.freshness IS DISTINCT FROM EXCLUDED.freshness;
""";
await using var command = connection.CreateCommand();
command.CommandText = sql;
@@ -336,6 +444,11 @@ internal sealed class UnifiedSearchIndexer : IUnifiedSearchIndexer
"embedding",
NpgsqlDbType.Array | NpgsqlDbType.Real,
chunk.Embedding is null ? Array.Empty<float>() : chunk.Embedding);
if (hasEmbeddingVectorColumn)
{
var vectorLiteral = chunk.Embedding is null ? (object)DBNull.Value : BuildVectorLiteral(chunk.Embedding);
command.Parameters.AddWithValue("embedding_vector", vectorLiteral);
}
command.Parameters.AddWithValue("metadata", NpgsqlDbType.Jsonb, chunk.Metadata.RootElement.GetRawText());
command.Parameters.AddWithValue("domain", chunk.Domain);
command.Parameters.AddWithValue("entity_key", (object?)chunk.EntityKey ?? DBNull.Value);
@@ -349,6 +462,32 @@ internal sealed class UnifiedSearchIndexer : IUnifiedSearchIndexer
return affectedRows;
}
private static async Task<bool> HasEmbeddingVectorColumnAsync(
NpgsqlConnection connection,
CancellationToken cancellationToken)
{
const string sql = """
SELECT EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'advisoryai'
AND table_name = 'kb_chunk'
AND column_name = 'embedding_vec'
);
""";
await using var command = connection.CreateCommand();
command.CommandText = sql;
command.CommandTimeout = 30;
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return result is bool value && value;
}
private static string BuildVectorLiteral(float[] values)
{
return "[" + string.Join(",", values.Select(static value => value.ToString("G9", CultureInfo.InvariantCulture))) + "]";
}
private static async Task EnsureDocumentExistsAsync(
NpgsqlConnection connection,
string docId,

119
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchService.cs
View File

@@ -179,7 +179,11 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
aggregateCoverage = MergeCoverage(aggregateCoverage, response.Coverage);
var cardCount = response.Cards.Count + (response.Overflow?.Cards.Count ?? 0);
var answer = response.ContextAnswer;
var viabilityState = DetermineSuggestionViabilityState(cardCount, answer, corpusAvailability);
var viabilityState = DetermineSuggestionViabilityState(
cardCount,
answer,
response.Coverage,
corpusAvailability);
results.Add(new SearchSuggestionViabilityResult(
Query: query,
@@ -191,7 +195,11 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
?? response.Overflow?.Cards.FirstOrDefault()?.Domain
?? response.Coverage?.Domains.FirstOrDefault(static domain => domain.HasVisibleResults)?.Domain
?? response.Coverage?.CurrentScopeDomain,
Reason: BuildSuggestionViabilityReason(answer, viabilityState, corpusAvailability),
Reason: BuildSuggestionViabilityReason(
answer,
viabilityState,
response.Coverage,
corpusAvailability),
ViabilityState: viabilityState,
ScopeReady: IsCurrentScopeReady(corpusAvailability)));
}
@@ -834,7 +842,8 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
UnifiedSearchOverflow? overflow)
{
var topCard = answerCards[0];
var scope = coverage?.CurrentScopeDomain
var currentScopeDomain = coverage?.CurrentScopeDomain;
var groundedScope = ResolveGroundedScopeDomain(coverage, topCard)
?? ResolveContextDomain(plan, [topCard], ambient)
?? topCard.Domain;
@@ -843,12 +852,19 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
return $"The highest-ranked results are close in score, so the answer blends evidence across {FormatDomainList(answerCards.Select(static card => card.Domain))}.";
}
if (overflow is not null)
if (overflow is not null && !string.IsNullOrWhiteSpace(currentScopeDomain))
{
return $"Current-scope weighting kept {DescribeDomain(scope)} first, while close related evidence from other domains remains visible below.";
return $"Current-scope weighting kept {DescribeDomain(currentScopeDomain)} first, while close related evidence from other domains remains visible below.";
}
return $"The top result is grounded in {DescribeDomain(scope)} evidence and aligns with the {plan.Intent} intent.";
if (!string.IsNullOrWhiteSpace(currentScopeDomain)
&& !string.Equals(groundedScope, currentScopeDomain, StringComparison.OrdinalIgnoreCase)
&& !HasVisibleCurrentScopeResults(coverage))
{
return $"The best grounded evidence came from {DescribeDomain(groundedScope)} after current-page weighting found no stronger match in {DescribeDomain(currentScopeDomain)}.";
}
return $"The top result is grounded in {DescribeDomain(groundedScope)} evidence and aligns with the {plan.Intent} intent.";
}
private static string BuildGroundedEvidence(
@@ -871,6 +887,13 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
return $"Grounded in {sourceCount} source(s) across {FormatDomainList(domains)} with {DescribeDomain(currentScopeDomain)} weighted first.";
}
if (coverage?.CurrentScopeDomain is { Length: > 0 } visibleScopeDomain
&& !HasVisibleCurrentScopeResults(coverage)
&& !string.Equals(answerCards[0].Domain, visibleScopeDomain, StringComparison.OrdinalIgnoreCase))
{
return $"Grounded in {sourceCount} source(s) across {FormatDomainList(domains)}. No visible evidence ranked in {DescribeDomain(visibleScopeDomain)}, so the answer falls back to {DescribeDomain(answerCards[0].Domain)}.";
}
return $"Grounded in {sourceCount} source(s) across {FormatDomainList(domains)}.";
}
@@ -1350,10 +1373,26 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
private static string DetermineSuggestionViabilityState(
int cardCount,
ContextAnswer? answer,
UnifiedSearchCoverage? coverage,
CorpusAvailabilitySnapshot corpusAvailability)
{
if (string.Equals(ResolveCorpusUnreadyCode(corpusAvailability), "current_scope_corpus_unready", StringComparison.Ordinal)
&& cardCount > 0
&& !HasVisibleCurrentScopeResults(coverage))
{
return "scope_unready";
}
if (cardCount > 0)
{
if (!string.IsNullOrWhiteSpace(coverage?.CurrentScopeDomain)
&& !string.Equals(ResolveCorpusUnreadyCode(corpusAvailability), "current_scope_corpus_unready", StringComparison.Ordinal)
&& !HasVisibleCurrentScopeResults(coverage)
&& CardCountExistsOutsideCurrentScope(answer, coverage))
{
return "outside_scope_only";
}
return "grounded";
}
@@ -1389,18 +1428,86 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
private static string BuildSuggestionViabilityReason(
ContextAnswer? answer,
string viabilityState,
UnifiedSearchCoverage? coverage,
CorpusAvailabilitySnapshot corpusAvailability)
{
return viabilityState switch
{
"grounded" => answer?.Reason ?? "Grounded evidence is available for this suggestion.",
"outside_scope_only" => BuildOutsideScopeOnlyReason(coverage),
"needs_clarification" => answer?.Reason ?? "The query is too broad to surface as a ready-made suggestion.",
"scope_unready" => BuildCorpusUnreadyReason(plan: null, ambient: null, corpusAvailability),
"corpus_unready" => BuildCorpusUnreadyReason(plan: null, ambient: null, corpusAvailability),
"no_match" when CardCountExistsOutsideCurrentScope(answer, coverage) =>
BuildOutsideScopeOnlyReason(coverage),
_ => answer?.Reason ?? "No grounded evidence matched this suggestion in the current corpus."
};
}
private static bool HasVisibleCurrentScopeResults(UnifiedSearchCoverage? coverage)
{
if (coverage is null || string.IsNullOrWhiteSpace(coverage.CurrentScopeDomain))
{
return false;
}
return coverage.Domains.Any(domain =>
domain.IsCurrentScope
&& domain.HasVisibleResults
&& domain.VisibleCardCount > 0);
}
private static string? ResolveGroundedScopeDomain(
UnifiedSearchCoverage? coverage,
EntityCard topCard)
{
if (coverage is null)
{
return null;
}
if (HasVisibleCurrentScopeResults(coverage)
&& !string.IsNullOrWhiteSpace(coverage.CurrentScopeDomain))
{
return coverage.CurrentScopeDomain;
}
if (!string.IsNullOrWhiteSpace(topCard.Domain))
{
return topCard.Domain;
}
return coverage.CurrentScopeDomain;
}
private static bool CardCountExistsOutsideCurrentScope(
ContextAnswer? answer,
UnifiedSearchCoverage? coverage)
{
if (!string.Equals(answer?.Status, "grounded", StringComparison.OrdinalIgnoreCase)
|| coverage is null
|| string.IsNullOrWhiteSpace(coverage.CurrentScopeDomain))
{
return false;
}
return !HasVisibleCurrentScopeResults(coverage)
&& coverage.Domains.Any(domain => domain.HasVisibleResults && !domain.IsCurrentScope);
}
private static string BuildOutsideScopeOnlyReason(UnifiedSearchCoverage? coverage)
{
var currentScopeDomain = coverage?.CurrentScopeDomain;
var visibleDomain = coverage?.Domains.FirstOrDefault(static domain => domain.HasVisibleResults)?.Domain;
if (!string.IsNullOrWhiteSpace(currentScopeDomain) && !string.IsNullOrWhiteSpace(visibleDomain))
{
return $"The current page did not return grounded evidence in {DescribeDomain(currentScopeDomain)}, but related evidence is available in {DescribeDomain(visibleDomain)}.";
}
return "The current page did not return grounded evidence, but related evidence is available outside the current scope.";
}
private static string? ResolveCorpusUnreadyCode(CorpusAvailabilitySnapshot corpusAvailability)
{
if (!corpusAvailability.Known)