feat(metrics): Add new histograms for chunk latency, results, and sources in AdvisoryAiMetrics
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
feat(telemetry): Record chunk latency, result count, and source count in AdvisoryAiTelemetry fix(endpoint): Include telemetry source count in advisory chunks endpoint response test(metrics): Enhance WebServiceEndpointsTests to validate new metrics for chunk latency, results, and sources refactor(tests): Update test utilities for Deno language analyzer tests chore(tests): Add performance tests for AdvisoryGuardrail with scenarios and blocked phrases docs: Archive Sprint 137 design document for scanner and surface enhancements
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Bundles;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using System.Linq;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Containers;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using System.Linq;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Deno;
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.Harness;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Golden;
|
||||
|
||||
@@ -15,9 +14,9 @@ public sealed class DenoAnalyzerGoldenTests
|
||||
var analyzers = new ILanguageAnalyzer[] { new DenoLanguageAnalyzer() };
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
|
||||
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixture, analyzers, cancellationToken).ConfigureAwait(false);
|
||||
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixture, analyzers, cancellationToken);
|
||||
var normalized = Normalize(json, fixture);
|
||||
var expected = await File.ReadAllTextAsync(golden, cancellationToken).ConfigureAwait(false);
|
||||
var expected = await File.ReadAllTextAsync(golden, cancellationToken);
|
||||
|
||||
normalized = normalized.TrimEnd();
|
||||
expected = expected.TrimEnd();
|
||||
@@ -25,7 +24,7 @@ public sealed class DenoAnalyzerGoldenTests
|
||||
if (!string.Equals(expected, normalized, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = golden + ".actual";
|
||||
await File.WriteAllTextAsync(actualPath, normalized, cancellationToken).ConfigureAwait(false);
|
||||
await File.WriteAllTextAsync(actualPath, normalized, cancellationToken);
|
||||
}
|
||||
|
||||
Assert.Equal(expected, normalized);
|
||||
|
||||
@@ -11,10 +11,15 @@ namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Observations;
|
||||
|
||||
public sealed class DenoLanguageAnalyzerObservationTests
|
||||
{
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
public DenoLanguageAnalyzerObservationTests(ITestOutputHelper output)
|
||||
=> _output = output;
|
||||
|
||||
[Fact]
|
||||
public async Task AnalyzerStoresObservationPayloadInAnalysisStoreAsync()
|
||||
{
|
||||
var (root, envDenoDir) = DenoWorkspaceTestFixture.Create();
|
||||
var (workspaceRoot, envDenoDir) = DenoWorkspaceTestFixture.Create();
|
||||
var previousDenoDir = Environment.GetEnvironmentVariable("DENO_DIR");
|
||||
|
||||
try
|
||||
@@ -23,7 +28,7 @@ public sealed class DenoLanguageAnalyzerObservationTests
|
||||
|
||||
var store = new ScanAnalysisStore();
|
||||
var context = new LanguageAnalyzerContext(
|
||||
root,
|
||||
workspaceRoot,
|
||||
TimeProvider.System,
|
||||
usageHints: null,
|
||||
services: null,
|
||||
@@ -40,31 +45,31 @@ public sealed class DenoLanguageAnalyzerObservationTests
|
||||
Assert.NotNull(payload.Metadata);
|
||||
Assert.True(payload.Metadata!.ContainsKey("deno.observation.hash"));
|
||||
|
||||
using var document = JsonDocument.Parse(payload.Content.Span);
|
||||
var root = document.RootElement;
|
||||
using var document = JsonDocument.Parse(payload.Content.ToArray());
|
||||
var observationRoot = document.RootElement;
|
||||
|
||||
var entrypoints = root.GetProperty("entrypoints").EnumerateArray().Select(element => element.GetString()).ToArray();
|
||||
var entrypoints = observationRoot.GetProperty("entrypoints").EnumerateArray().Select(element => element.GetString()).ToArray();
|
||||
Assert.Contains("src/main.ts", entrypoints);
|
||||
|
||||
var capabilities = root.GetProperty("capabilities").EnumerateArray().ToArray();
|
||||
var capabilities = observationRoot.GetProperty("capabilities").EnumerateArray().ToArray();
|
||||
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.deno.ffi");
|
||||
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.node.worker_threads");
|
||||
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.node.fs");
|
||||
|
||||
var dynamicImports = root.GetProperty("dynamicImports").EnumerateArray().Select(element => element.GetProperty("specifier").GetString()).ToArray();
|
||||
var dynamicImports = observationRoot.GetProperty("dynamicImports").EnumerateArray().Select(element => element.GetProperty("specifier").GetString()).ToArray();
|
||||
Assert.Contains("https://cdn.example.com/dynamic/mod.ts", dynamicImports);
|
||||
|
||||
var literalFetches = root.GetProperty("literalFetches").EnumerateArray().Select(element => element.GetProperty("url").GetString()).ToArray();
|
||||
var literalFetches = observationRoot.GetProperty("literalFetches").EnumerateArray().Select(element => element.GetProperty("url").GetString()).ToArray();
|
||||
Assert.Contains("https://api.example.com/data.json", literalFetches);
|
||||
|
||||
var bundles = root.GetProperty("bundles").EnumerateArray().ToArray();
|
||||
var bundles = observationRoot.GetProperty("bundles").EnumerateArray().ToArray();
|
||||
Assert.Contains(bundles, bundle => bundle.GetProperty("type").GetString() == "eszip");
|
||||
Assert.Contains(bundles, bundle => bundle.GetProperty("type").GetString() == "deno-compile");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Environment.SetEnvironmentVariable("DENO_DIR", previousDenoDir);
|
||||
DenoWorkspaceTestFixture.Cleanup(root);
|
||||
DenoWorkspaceTestFixture.Cleanup(workspaceRoot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\\StellaOps.Scanner.Analyzers.Lang.Tests\\StellaOps.Scanner.Analyzers.Lang.Tests.csproj" />
|
||||
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Scanner.Analyzers.Lang\\StellaOps.Scanner.Analyzers.Lang.csproj" />
|
||||
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Scanner.Analyzers.Lang.Deno\\StellaOps.Scanner.Analyzers.Lang.Deno.csproj" />
|
||||
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Scanner.Core\\StellaOps.Scanner.Core.csproj" />
|
||||
@@ -39,4 +38,10 @@
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\\StellaOps.Scanner.Analyzers.Lang.Tests\\Fixtures\\**\\*"
|
||||
Link="Fixtures\\%(RecursiveDir)%(Filename)%(Extension)"
|
||||
CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
internal static class LanguageAnalyzerTestHarness
|
||||
{
|
||||
public static async Task<string> RunToJsonAsync(
|
||||
string fixturePath,
|
||||
IEnumerable<ILanguageAnalyzer> analyzers,
|
||||
CancellationToken cancellationToken = default,
|
||||
LanguageUsageHints? usageHints = null,
|
||||
IServiceProvider? services = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(fixturePath))
|
||||
{
|
||||
throw new ArgumentException("Fixture path is required", nameof(fixturePath));
|
||||
}
|
||||
|
||||
var engine = new LanguageAnalyzerEngine(analyzers ?? Array.Empty<ILanguageAnalyzer>());
|
||||
var context = new LanguageAnalyzerContext(fixturePath, TimeProvider.System, usageHints, services);
|
||||
var result = await engine.AnalyzeAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
return result.ToJson(indent: true);
|
||||
}
|
||||
|
||||
public static async Task AssertDeterministicAsync(
|
||||
string fixturePath,
|
||||
string goldenPath,
|
||||
IEnumerable<ILanguageAnalyzer> analyzers,
|
||||
CancellationToken cancellationToken = default,
|
||||
LanguageUsageHints? usageHints = null,
|
||||
IServiceProvider? services = null)
|
||||
{
|
||||
var actual = await RunToJsonAsync(fixturePath, analyzers, cancellationToken, usageHints, services).ConfigureAwait(false);
|
||||
var expected = await File.ReadAllTextAsync(goldenPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
actual = NormalizeLineEndings(actual).TrimEnd();
|
||||
expected = NormalizeLineEndings(expected).TrimEnd();
|
||||
|
||||
if (!string.Equals(expected, actual, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = goldenPath + ".actual";
|
||||
var directory = Path.GetDirectoryName(actualPath);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
await File.WriteAllTextAsync(actualPath, actual, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Assert.Equal(expected, actual);
|
||||
}
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestUtilities;
|
||||
|
||||
internal static class TestPaths
|
||||
{
|
||||
public static string ResolveFixture(params string[] segments)
|
||||
{
|
||||
var baseDirectory = AppContext.BaseDirectory;
|
||||
var parts = new List<string> { baseDirectory };
|
||||
parts.Add("Fixtures");
|
||||
if (segments is not null && segments.Length > 0)
|
||||
{
|
||||
parts.AddRange(segments);
|
||||
}
|
||||
|
||||
return Path.GetFullPath(Path.Combine(parts.ToArray()));
|
||||
}
|
||||
|
||||
public static string CreateTemporaryDirectory()
|
||||
{
|
||||
var root = Path.Combine(AppContext.BaseDirectory, "tmp", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
public static void SafeDelete(string directory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directory) || !Directory.Exists(directory))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// best-effort cleanup; avoid masking upstream test failures
|
||||
}
|
||||
}
|
||||
|
||||
public static string ResolveProjectRoot()
|
||||
{
|
||||
var directory = AppContext.BaseDirectory;
|
||||
while (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
var matches = Directory.EnumerateFiles(
|
||||
directory,
|
||||
"StellaOps.Scanner.Analyzers.Lang.Deno.Tests.csproj",
|
||||
SearchOption.TopDirectoryOnly);
|
||||
if (matches.Any())
|
||||
{
|
||||
return directory;
|
||||
}
|
||||
|
||||
directory = Path.GetDirectoryName(directory) ?? string.Empty;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Unable to locate project root.");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user