stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

Tighten unified search ranking and optional telemetry

Browse Source
This commit is contained in:
master
2026-03-07 20:29:44 +02:00
parent f23ca585d4
commit 55701483ea
8 changed files with 277 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchService.cs
View File

@@ -24,8 +24,8 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
private const int MaxSuggestionViabilityQueries = 6;
private const int MaxOverflowCards = 4;
private const int CoverageCandidateWindow = 24;
private const double OverflowScoreBandRatio = 0.15d;
private const double BlendedAnswerScoreBandRatio = 0.12d;
private const double OverflowScoreBandRatio = 0.04d;
private const double BlendedAnswerScoreBandRatio = 0.025d;
private readonly KnowledgeSearchOptions _options;
private readonly UnifiedSearchOptions _unifiedOptions;
private readonly IKnowledgeSearchStore _store;
@@ -2269,7 +2269,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
private void EmitTelemetry(QueryPlan plan, UnifiedSearchResponse response, string tenant)
{
if (_telemetrySink is null)
if (_telemetrySink is null || !_options.SearchTelemetryEnabled)
{
return;
}

51
src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/SearchAnalyticsServiceTests.cs Normal file
View File

@@ -0,0 +1,51 @@
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.AdvisoryAI.KnowledgeSearch;
using StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.UnifiedSearch;
public sealed class SearchAnalyticsServiceTests
{
[Fact]
public async Task RecordEventsAndPopularityRemainDisabledWhileHistoryStillWorks()
{
var options = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = string.Empty,
SearchTelemetryEnabled = false
});
var service = new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
await service.RecordEventsAsync(
[
new SearchAnalyticsEvent(
TenantId: "tenant-disabled",
EventType: "click",
Query: "database connectivity",
UserId: "user-1",
EntityKey: "knowledge:doctor:db",
Domain: "knowledge",
Position: 0)
]);
await service.RecordHistoryAsync(
"tenant-disabled",
"user-1",
"database connectivity",
2);
var history = await service.GetHistoryAsync("tenant-disabled", "user-1");
var popularity = await service.GetPopularityMapAsync("tenant-disabled", 30);
var events = service.GetFallbackEventsSnapshot("tenant-disabled", TimeSpan.FromMinutes(1));
events.Should().BeEmpty();
popularity.Should().BeEmpty();
history.Should().ContainSingle();
history[0].Query.Should().Be("database connectivity");
history[0].ResultCount.Should().Be(2);
}
}

38
src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/SearchQualityMonitorTests.cs Normal file
View File

@@ -0,0 +1,38 @@
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.AdvisoryAI.KnowledgeSearch;
using StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.UnifiedSearch;
public sealed class SearchQualityMonitorTests
{
[Fact]
public async Task StoreFeedbackDoesNotPersistWhenTelemetryIsDisabled()
{
var options = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = string.Empty,
SearchTelemetryEnabled = false
});
var analytics = new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
var monitor = new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance, analytics);
await monitor.StoreFeedbackAsync(new SearchFeedbackEntry
{
TenantId = "tenant-disabled",
UserId = "user-1",
Query = "database connectivity",
EntityKey = "knowledge:doctor:db",
Domain = "knowledge",
Position = 0,
Signal = "not_helpful"
});
monitor.GetFallbackFeedbackSnapshot("tenant-disabled", TimeSpan.FromMinutes(1))
.Should().BeEmpty();
}
}

179
src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/UnifiedSearchServiceTests.cs
View File

@@ -138,6 +138,58 @@ public sealed class UnifiedSearchServiceTests
result.ContextAnswer!.Code.Should().Be("retrieved_scope_weighted_evidence");
}
[Fact]
public async Task SearchAsync_suppresses_overflow_when_current_scope_result_has_clear_lead()
{
var doctorEmbedding = new float[64];
doctorEmbedding[0] = 0.7f;
doctorEmbedding[1] = 0.2f;
var doctorRow = MakeRow(
"chunk-doctor-dominant-scope",
"doctor_check",
"PostgreSQL connectivity",
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"),
embedding: doctorEmbedding,
snippet: "Doctor knowledge confirms the database endpoint is down.");
var findingRow = MakeRow(
"chunk-finding-weak-overflow",
"finding",
"CVE-2026-9001 in postgres sidecar",
JsonDocument.Parse("{\"domain\":\"findings\",\"cveId\":\"CVE-2026-9001\",\"severity\":\"high\"}"),
snippet: "A related finding exists, but it is secondary to the current page problem.");
var storeMock = new Mock<IKnowledgeSearchStore>();
storeMock.Setup(s => s.SearchFtsAsync(
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow, findingRow });
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
var service = CreateService(storeMock: storeMock);
var result = await service.SearchAsync(
new UnifiedSearchRequest(
"database connectivity",
Ambient: new AmbientContext
{
CurrentRoute = "/ops/operations/doctor"
}),
CancellationToken.None);
result.Cards.Should().ContainSingle();
result.Cards[0].Domain.Should().Be("knowledge");
result.Overflow.Should().BeNull();
result.Coverage.Should().NotBeNull();
result.Coverage!.Domains.Should().Contain(domain =>
domain.Domain == "findings" && !domain.HasVisibleResults);
result.ContextAnswer.Should().NotBeNull();
result.ContextAnswer!.Code.Should().Be("retrieved_evidence");
result.ContextAnswer.Citations.Should().ContainSingle();
}
[Fact]
public async Task SearchAsync_blends_close_top_results_into_one_answer()
{
@@ -177,6 +229,49 @@ public sealed class UnifiedSearchServiceTests
result.ContextAnswer.Citations.Should().HaveCount(2);
}
[Fact]
public async Task SearchAsync_keeps_single_dominant_answer_when_top_result_has_clear_lead()
{
var doctorEmbedding = new float[64];
doctorEmbedding[0] = 0.6f;
doctorEmbedding[1] = 0.3f;
var doctorRow = MakeRow(
"chunk-doctor-dominant-answer",
"doctor_check",
"PostgreSQL connectivity",
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"),
embedding: doctorEmbedding,
snippet: "Database connectivity is degraded.");
var guideRow = MakeRow(
"chunk-guide-secondary-answer",
"md_section",
"Database failover playbook",
JsonDocument.Parse("{\"domain\":\"knowledge\",\"path\":\"docs/database-failover.md\"}"),
snippet: "Failover guidance covers the same connectivity symptoms.");
var storeMock = new Mock<IKnowledgeSearchStore>();
storeMock.Setup(s => s.SearchFtsAsync(
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow, guideRow });
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
var service = CreateService(storeMock: storeMock);
var result = await service.SearchAsync(
new UnifiedSearchRequest("database connectivity", IncludeSynthesis: false),
CancellationToken.None);
result.ContextAnswer.Should().NotBeNull();
result.ContextAnswer!.Code.Should().Be("retrieved_evidence");
result.ContextAnswer.Summary.Should().NotContain("Database failover playbook");
result.ContextAnswer.Citations.Should().ContainSingle();
result.ContextAnswer.Citations![0].Title.Should().Be("PostgreSQL connectivity");
}
[Fact]
public async Task SearchAsync_returns_clarify_context_answer_for_broad_query_without_matches()
{
@@ -224,7 +319,7 @@ public sealed class UnifiedSearchServiceTests
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = "Host=localhost;Database=test"
ConnectionString = "Host=localhost;Database=test;Timeout=1;Command Timeout=1"
});
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
@@ -263,7 +358,7 @@ public sealed class UnifiedSearchServiceTests
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = "Host=localhost;Database=test"
ConnectionString = "Host=localhost;Database=test;Timeout=1;Command Timeout=1"
});
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
@@ -313,13 +408,67 @@ public sealed class UnifiedSearchServiceTests
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-456"));
}
[Fact]
public async Task SearchAsync_does_not_record_answer_frame_analytics_or_emit_sink_when_telemetry_disabled()
{
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = "Host=localhost;Database=test;Timeout=1;Command Timeout=1",
SearchTelemetryEnabled = false
});
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
var telemetrySink = new RecordingTelemetrySink();
var doctorRow = MakeRow(
"chunk-doctor-telemetry-off",
"doctor_check",
"PostgreSQL connectivity",
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"));
var storeMock = new Mock<IKnowledgeSearchStore>();
storeMock.Setup(s => s.SearchFtsAsync(
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.ReturnsAsync([]);
var service = CreateService(
storeMock: storeMock,
analyticsService: analyticsService,
qualityMonitor: qualityMonitor,
configureOptions: options => options.SearchTelemetryEnabled = false,
telemetrySink: telemetrySink);
var result = await service.SearchAsync(
new UnifiedSearchRequest(
"database connectivity",
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
Ambient: new AmbientContext
{
CurrentRoute = "/ops/operations/doctor",
SessionId = "session-off"
}),
CancellationToken.None);
result.ContextAnswer.Should().NotBeNull();
result.ContextAnswer!.Status.Should().Be("grounded");
analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1))
.Should().BeEmpty();
telemetrySink.Events.Should().BeEmpty();
}
[Fact]
public async Task EvaluateSuggestionsAsync_returns_viability_without_recording_answer_frame_analytics()
{
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = "Host=localhost;Database=test"
ConnectionString = "Host=localhost;Database=test;Timeout=1;Command Timeout=1"
});
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
@@ -1086,19 +1235,23 @@ public sealed class UnifiedSearchServiceTests
Mock<IKnowledgeSearchStore>? storeMock = null,
UnifiedSearchOptions? unifiedOptions = null,
SearchAnalyticsService? analyticsService = null,
SearchQualityMonitor? qualityMonitor = null)
SearchQualityMonitor? qualityMonitor = null,
Action<KnowledgeSearchOptions>? configureOptions = null,
IUnifiedSearchTelemetrySink? telemetrySink = null)
{
var options = Options.Create(new KnowledgeSearchOptions
var knowledgeOptions = new KnowledgeSearchOptions
{
Enabled = enabled,
ConnectionString = enabled ? "Host=localhost;Database=test" : "",
ConnectionString = enabled ? "Host=localhost;Database=test;Timeout=1;Command Timeout=1" : "",
DefaultTopK = 10,
VectorDimensions = 64,
FtsCandidateCount = 120,
VectorScanLimit = 100,
VectorCandidateCount = 50,
QueryTimeoutMs = 3000
});
};
configureOptions?.Invoke(knowledgeOptions);
var options = Options.Create(knowledgeOptions);
var wrappedUnifiedOptions = Options.Create(unifiedOptions ?? new UnifiedSearchOptions());
storeMock ??= new Mock<IKnowledgeSearchStore>();
@@ -1133,10 +1286,20 @@ public sealed class UnifiedSearchServiceTests
entityAliasService.Object,
logger,
timeProvider,
telemetrySink: null,
telemetrySink: telemetrySink,
unifiedOptions: wrappedUnifiedOptions);
}
private sealed class RecordingTelemetrySink : IUnifiedSearchTelemetrySink
{
public List<UnifiedSearchTelemetryEvent> Events { get; } = [];
public void Record(UnifiedSearchTelemetryEvent telemetryEvent)
{
Events.Add(telemetryEvent);
}
}
private static Mock<IKnowledgeSearchStore> CreateEmptyStoreMock()
{
var storeMock = new Mock<IKnowledgeSearchStore>();