feat(metrics): Add new histograms for chunk latency, results, and sources in AdvisoryAiMetrics
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

feat(telemetry): Record chunk latency, result count, and source count in AdvisoryAiTelemetry

fix(endpoint): Include telemetry source count in advisory chunks endpoint response

test(metrics): Enhance WebServiceEndpointsTests to validate new metrics for chunk latency, results, and sources

refactor(tests): Update test utilities for Deno language analyzer tests

chore(tests): Add performance tests for AdvisoryGuardrail with scenarios and blocked phrases

docs: Archive Sprint 137 design document for scanner and surface enhancements
This commit is contained in:
master
2025-11-10 22:26:43 +02:00
parent 56c687253f
commit b059bc7675
22 changed files with 427 additions and 37 deletions

View File

@@ -1,6 +1,7 @@
using StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Bundles;

View File

@@ -1,7 +1,7 @@
using System.Linq;
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Containers;

View File

@@ -1,7 +1,7 @@
using System.Linq;
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Deno;

View File

@@ -1,7 +1,6 @@
using StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Analyzers.Lang.Deno;
using StellaOps.Scanner.Analyzers.Lang.Tests.Harness;
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Golden;
@@ -15,9 +14,9 @@ public sealed class DenoAnalyzerGoldenTests
var analyzers = new ILanguageAnalyzer[] { new DenoLanguageAnalyzer() };
var cancellationToken = TestContext.Current.CancellationToken;
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixture, analyzers, cancellationToken).ConfigureAwait(false);
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixture, analyzers, cancellationToken);
var normalized = Normalize(json, fixture);
var expected = await File.ReadAllTextAsync(golden, cancellationToken).ConfigureAwait(false);
var expected = await File.ReadAllTextAsync(golden, cancellationToken);
normalized = normalized.TrimEnd();
expected = expected.TrimEnd();
@@ -25,7 +24,7 @@ public sealed class DenoAnalyzerGoldenTests
if (!string.Equals(expected, normalized, StringComparison.Ordinal))
{
var actualPath = golden + ".actual";
await File.WriteAllTextAsync(actualPath, normalized, cancellationToken).ConfigureAwait(false);
await File.WriteAllTextAsync(actualPath, normalized, cancellationToken);
}
Assert.Equal(expected, normalized);

View File

@@ -11,10 +11,15 @@ namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Observations;
public sealed class DenoLanguageAnalyzerObservationTests
{
private readonly ITestOutputHelper _output;
public DenoLanguageAnalyzerObservationTests(ITestOutputHelper output)
=> _output = output;
[Fact]
public async Task AnalyzerStoresObservationPayloadInAnalysisStoreAsync()
{
var (root, envDenoDir) = DenoWorkspaceTestFixture.Create();
var (workspaceRoot, envDenoDir) = DenoWorkspaceTestFixture.Create();
var previousDenoDir = Environment.GetEnvironmentVariable("DENO_DIR");
try
@@ -23,7 +28,7 @@ public sealed class DenoLanguageAnalyzerObservationTests
var store = new ScanAnalysisStore();
var context = new LanguageAnalyzerContext(
root,
workspaceRoot,
TimeProvider.System,
usageHints: null,
services: null,
@@ -40,31 +45,31 @@ public sealed class DenoLanguageAnalyzerObservationTests
Assert.NotNull(payload.Metadata);
Assert.True(payload.Metadata!.ContainsKey("deno.observation.hash"));
using var document = JsonDocument.Parse(payload.Content.Span);
var root = document.RootElement;
using var document = JsonDocument.Parse(payload.Content.ToArray());
var observationRoot = document.RootElement;
var entrypoints = root.GetProperty("entrypoints").EnumerateArray().Select(element => element.GetString()).ToArray();
var entrypoints = observationRoot.GetProperty("entrypoints").EnumerateArray().Select(element => element.GetString()).ToArray();
Assert.Contains("src/main.ts", entrypoints);
var capabilities = root.GetProperty("capabilities").EnumerateArray().ToArray();
var capabilities = observationRoot.GetProperty("capabilities").EnumerateArray().ToArray();
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.deno.ffi");
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.node.worker_threads");
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.node.fs");
var dynamicImports = root.GetProperty("dynamicImports").EnumerateArray().Select(element => element.GetProperty("specifier").GetString()).ToArray();
var dynamicImports = observationRoot.GetProperty("dynamicImports").EnumerateArray().Select(element => element.GetProperty("specifier").GetString()).ToArray();
Assert.Contains("https://cdn.example.com/dynamic/mod.ts", dynamicImports);
var literalFetches = root.GetProperty("literalFetches").EnumerateArray().Select(element => element.GetProperty("url").GetString()).ToArray();
var literalFetches = observationRoot.GetProperty("literalFetches").EnumerateArray().Select(element => element.GetProperty("url").GetString()).ToArray();
Assert.Contains("https://api.example.com/data.json", literalFetches);
var bundles = root.GetProperty("bundles").EnumerateArray().ToArray();
var bundles = observationRoot.GetProperty("bundles").EnumerateArray().ToArray();
Assert.Contains(bundles, bundle => bundle.GetProperty("type").GetString() == "eszip");
Assert.Contains(bundles, bundle => bundle.GetProperty("type").GetString() == "deno-compile");
}
finally
{
Environment.SetEnvironmentVariable("DENO_DIR", previousDenoDir);
DenoWorkspaceTestFixture.Cleanup(root);
DenoWorkspaceTestFixture.Cleanup(workspaceRoot);
}
}
}

View File

@@ -24,7 +24,6 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\\StellaOps.Scanner.Analyzers.Lang.Tests\\StellaOps.Scanner.Analyzers.Lang.Tests.csproj" />
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Scanner.Analyzers.Lang\\StellaOps.Scanner.Analyzers.Lang.csproj" />
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Scanner.Analyzers.Lang.Deno\\StellaOps.Scanner.Analyzers.Lang.Deno.csproj" />
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Scanner.Core\\StellaOps.Scanner.Core.csproj" />
@@ -39,4 +38,10 @@
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
<ItemGroup>
<None Include="..\\StellaOps.Scanner.Analyzers.Lang.Tests\\Fixtures\\**\\*"
Link="Fixtures\\%(RecursiveDir)%(Filename)%(Extension)"
CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
</Project>

View File

@@ -1,5 +1,5 @@
using StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;

View File

@@ -0,0 +1,56 @@
using StellaOps.Scanner.Analyzers.Lang;
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
internal static class LanguageAnalyzerTestHarness
{
public static async Task<string> RunToJsonAsync(
string fixturePath,
IEnumerable<ILanguageAnalyzer> analyzers,
CancellationToken cancellationToken = default,
LanguageUsageHints? usageHints = null,
IServiceProvider? services = null)
{
if (string.IsNullOrWhiteSpace(fixturePath))
{
throw new ArgumentException("Fixture path is required", nameof(fixturePath));
}
var engine = new LanguageAnalyzerEngine(analyzers ?? Array.Empty<ILanguageAnalyzer>());
var context = new LanguageAnalyzerContext(fixturePath, TimeProvider.System, usageHints, services);
var result = await engine.AnalyzeAsync(context, cancellationToken).ConfigureAwait(false);
return result.ToJson(indent: true);
}
public static async Task AssertDeterministicAsync(
string fixturePath,
string goldenPath,
IEnumerable<ILanguageAnalyzer> analyzers,
CancellationToken cancellationToken = default,
LanguageUsageHints? usageHints = null,
IServiceProvider? services = null)
{
var actual = await RunToJsonAsync(fixturePath, analyzers, cancellationToken, usageHints, services).ConfigureAwait(false);
var expected = await File.ReadAllTextAsync(goldenPath, cancellationToken).ConfigureAwait(false);
actual = NormalizeLineEndings(actual).TrimEnd();
expected = NormalizeLineEndings(expected).TrimEnd();
if (!string.Equals(expected, actual, StringComparison.Ordinal))
{
var actualPath = goldenPath + ".actual";
var directory = Path.GetDirectoryName(actualPath);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
await File.WriteAllTextAsync(actualPath, actual, cancellationToken).ConfigureAwait(false);
}
Assert.Equal(expected, actual);
}
private static string NormalizeLineEndings(string value)
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
}

View File

@@ -0,0 +1,61 @@
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
internal static class TestPaths
{
public static string ResolveFixture(params string[] segments)
{
var baseDirectory = AppContext.BaseDirectory;
var parts = new List<string> { baseDirectory };
parts.Add("Fixtures");
if (segments is not null && segments.Length > 0)
{
parts.AddRange(segments);
}
return Path.GetFullPath(Path.Combine(parts.ToArray()));
}
public static string CreateTemporaryDirectory()
{
var root = Path.Combine(AppContext.BaseDirectory, "tmp", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(root);
return root;
}
public static void SafeDelete(string directory)
{
if (string.IsNullOrWhiteSpace(directory) || !Directory.Exists(directory))
{
return;
}
try
{
Directory.Delete(directory, recursive: true);
}
catch
{
// best-effort cleanup; avoid masking upstream test failures
}
}
public static string ResolveProjectRoot()
{
var directory = AppContext.BaseDirectory;
while (!string.IsNullOrEmpty(directory))
{
var matches = Directory.EnumerateFiles(
directory,
"StellaOps.Scanner.Analyzers.Lang.Deno.Tests.csproj",
SearchOption.TopDirectoryOnly);
if (matches.Any())
{
return directory;
}
directory = Path.GetDirectoryName(directory) ?? string.Empty;
}
throw new InvalidOperationException("Unable to locate project root.");
}
}