// -----------------------------------------------------------------------------
// SmartDiffPerfSmokeTests.cs
// Sprint: SPRINT_5100_0009_0001 - Scanner Module Test Implementation
// Task: SCANNER-5100-024 - Add perf smoke tests for smart diff (2× regression gate)
// Description: Performance smoke tests for SmartDiff with 2× regression gate.
// -----------------------------------------------------------------------------
using System.Diagnostics;
using System.Text.Json;
using FluentAssertions;
using Xunit;
using Xunit.Abstractions;
namespace StellaOps.Scanner.SmartDiffTests.Benchmarks;
///
/// Performance smoke tests for SmartDiff calculation.
/// These tests enforce a 2× regression gate: if performance regresses to more than
/// twice the baseline, the test fails.
///
/// Baselines are conservative estimates based on expected behavior.
/// Run periodically in CI to detect performance regressions.
///
[Trait("Category", "Perf")]
[Trait("Category", "PERF")]
[Trait("Category", "Smoke")]
public sealed class SmartDiffPerfSmokeTests
{
private readonly ITestOutputHelper _output;
// Regression gate multiplier: 2× means test fails if time exceeds 2× baseline
private const double RegressionGateMultiplier = 2.0;
// Baselines (in milliseconds) - conservative estimates
private const long BaselineSmallDiffMs = 25; // 50 pkgs, 10 vulns
private const long BaselineMediumDiffMs = 100; // 500 pkgs, 100 vulns
private const long BaselineLargeDiffMs = 500; // 5000 pkgs, 1000 vulns
private const long BaselineXLargeDiffMs = 2000; // 10000 pkgs, 2000 vulns
private const long BaselineSarifGenerationMs = 50; // SARIF output generation
private const long BaselineScoringSingleMs = 5; // Single finding scoring
private const long BaselineScoringBatchMs = 100; // Batch scoring (100 findings)
public SmartDiffPerfSmokeTests(ITestOutputHelper output)
{
_output = output;
}
#region Diff Computation Performance
[Fact]
public void SmallDiff_Computation_Under2xBaseline()
{
// Arrange
const int packageCount = 50;
const int vulnCount = 10;
var baseline = BaselineSmallDiffMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanData(packageCount, vulnCount, seed: 42);
var currentScan = GenerateScanData(packageCount + 5, vulnCount + 2, seed: 43);
// Warm up
_ = ComputeDiff(baselineScan, currentScan);
// Act
var sw = Stopwatch.StartNew();
var diff = ComputeDiff(baselineScan, currentScan);
sw.Stop();
// Log
_output.WriteLine($"Small diff ({packageCount} pkgs, {vulnCount} vulns): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
_output.WriteLine($"Added: {diff.Added.Count}, Removed: {diff.Removed.Count}");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"Small diff exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
[Fact]
public void MediumDiff_Computation_Under2xBaseline()
{
// Arrange
const int packageCount = 500;
const int vulnCount = 100;
var baseline = BaselineMediumDiffMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanData(packageCount, vulnCount, seed: 42);
var currentScan = GenerateScanData(packageCount + 20, vulnCount + 10, seed: 43);
// Warm up
_ = ComputeDiff(baselineScan, currentScan);
// Act
var sw = Stopwatch.StartNew();
var diff = ComputeDiff(baselineScan, currentScan);
sw.Stop();
// Log
_output.WriteLine($"Medium diff ({packageCount} pkgs, {vulnCount} vulns): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"Medium diff exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
[Fact]
public void LargeDiff_Computation_Under2xBaseline()
{
// Arrange
const int packageCount = 5000;
const int vulnCount = 1000;
var baseline = BaselineLargeDiffMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanData(packageCount, vulnCount, seed: 42);
var currentScan = GenerateScanData(packageCount + 100, vulnCount + 50, seed: 43);
// Warm up
_ = ComputeDiff(baselineScan, currentScan);
// Act
var sw = Stopwatch.StartNew();
var diff = ComputeDiff(baselineScan, currentScan);
sw.Stop();
// Log
_output.WriteLine($"Large diff ({packageCount} pkgs, {vulnCount} vulns): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"Large diff exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
[Fact]
public void XLargeDiff_Computation_Under2xBaseline()
{
// Arrange
const int packageCount = 10000;
const int vulnCount = 2000;
var baseline = BaselineXLargeDiffMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanData(packageCount, vulnCount, seed: 42);
var currentScan = GenerateScanData(packageCount + 200, vulnCount + 100, seed: 43);
// Warm up (smaller)
_ = ComputeDiff(
GenerateScanData(1000, 200, seed: 100),
GenerateScanData(1050, 220, seed: 101));
// Act
var sw = Stopwatch.StartNew();
var diff = ComputeDiff(baselineScan, currentScan);
sw.Stop();
// Log
_output.WriteLine($"XLarge diff ({packageCount} pkgs, {vulnCount} vulns): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"XLarge diff exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
#endregion
#region SARIF Generation Performance
[Fact]
public void SarifGeneration_Under2xBaseline()
{
// Arrange
var baseline = BaselineSarifGenerationMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanData(500, 100, seed: 42);
var currentScan = GenerateScanData(550, 120, seed: 43);
var diff = ComputeDiff(baselineScan, currentScan);
// Warm up
_ = GenerateSarif(diff);
// Act
var sw = Stopwatch.StartNew();
var sarif = GenerateSarif(diff);
sw.Stop();
// Log
_output.WriteLine($"SARIF generation ({diff.Added.Count} added, {diff.Removed.Count} removed): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Output size: {sarif.Length / 1024.0:F1}KB");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"SARIF generation exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
[Fact]
public void SarifGeneration_LargeDiff_Under2xBaseline()
{
// Arrange
var baseline = BaselineSarifGenerationMs * 5; // Scale up for larger diff
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanData(5000, 1000, seed: 42);
var currentScan = GenerateScanData(5200, 1100, seed: 43);
var diff = ComputeDiff(baselineScan, currentScan);
// Act
var sw = Stopwatch.StartNew();
var sarif = GenerateSarif(diff);
sw.Stop();
// Log
_output.WriteLine($"SARIF generation large ({diff.Added.Count} added): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Output size: {sarif.Length / 1024.0:F1}KB");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"Large SARIF generation exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
#endregion
#region Scoring Performance
[Fact]
public void SingleFindingScoring_Under2xBaseline()
{
// Arrange
var baseline = BaselineScoringSingleMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var finding = CreateFinding("CVE-2024-1234", "HIGH", true, "executed");
// Warm up
for (int i = 0; i < 100; i++) _ = ScoreFinding(finding);
// Act - run many iterations for accurate measurement
const int iterations = 1000;
var sw = Stopwatch.StartNew();
for (int i = 0; i < iterations; i++)
{
_ = ScoreFinding(finding);
}
sw.Stop();
var avgMs = sw.Elapsed.TotalMilliseconds / iterations;
// Log
_output.WriteLine($"Single finding scoring: {avgMs:F4}ms average over {iterations} iterations");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
avgMs.Should().BeLessThanOrEqualTo(threshold,
$"Single scoring exceeded 2× regression gate ({avgMs:F4}ms > {threshold}ms)");
}
[Fact]
public void BatchScoring_Under2xBaseline()
{
// Arrange
const int findingCount = 100;
var baseline = BaselineScoringBatchMs;
var threshold = (long)(baseline * RegressionGateMultiplier);
var findings = Enumerable.Range(0, findingCount)
.Select(i => CreateFinding($"CVE-2024-{i:D4}",
i % 4 == 0 ? "CRITICAL" : i % 4 == 1 ? "HIGH" : i % 4 == 2 ? "MEDIUM" : "LOW",
i % 3 != 0,
i % 2 == 0 ? "executed" : "called"))
.ToList();
// Warm up
_ = ScoreBatch(findings);
// Act
var sw = Stopwatch.StartNew();
var scores = ScoreBatch(findings);
sw.Stop();
// Log
_output.WriteLine($"Batch scoring ({findingCount} findings): {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"Batch scoring exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
scores.Should().HaveCount(findingCount);
}
#endregion
#region Scaling Behavior
[Fact]
public void DiffComputation_ScalesLinearlyWithSize()
{
// Arrange - test that diff computation is O(n) not O(n²)
var sizes = new[] { 100, 500, 1000, 2000 };
var times = new List<(int size, long ms)>();
foreach (var size in sizes)
{
var baselineScan = GenerateScanData(size, size / 5, seed: 42);
var currentScan = GenerateScanData(size + size / 10, size / 5 + size / 50, seed: 43);
var sw = Stopwatch.StartNew();
_ = ComputeDiff(baselineScan, currentScan);
sw.Stop();
times.Add((size, sw.ElapsedMilliseconds));
_output.WriteLine($"Size {size}: {sw.ElapsedMilliseconds}ms");
}
// Assert - verify roughly linear scaling (within 4× of linear for O(n))
// If 2× input takes more than 4× time, it's superlinear
for (int i = 1; i < times.Count; i++)
{
var sizeRatio = times[i].size / (double)times[i - 1].size;
var timeRatio = times[i].ms / Math.Max(1.0, times[i - 1].ms);
var scaleFactor = timeRatio / sizeRatio;
_output.WriteLine($"Size ratio: {sizeRatio:F1}×, Time ratio: {timeRatio:F1}×, Scale factor: {scaleFactor:F2}");
// Allow some variance, but should be better than O(n²)
scaleFactor.Should().BeLessThan(2.5,
$"Diff computation shows non-linear scaling at size {times[i].size}");
}
}
[Fact]
public void DiffComputation_WithReachabilityFlips_UnderBaseline()
{
// Arrange - test performance when reachability changes
const int packageCount = 1000;
const int vulnCount = 200;
var baseline = 150L; // ms
var threshold = (long)(baseline * RegressionGateMultiplier);
var baselineScan = GenerateScanDataWithReachability(packageCount, vulnCount, reachableRatio: 0.3, seed: 42);
var currentScan = GenerateScanDataWithReachability(packageCount, vulnCount, reachableRatio: 0.5, seed: 42);
// Warm up
_ = ComputeDiffWithReachability(baselineScan, currentScan);
// Act
var sw = Stopwatch.StartNew();
var diff = ComputeDiffWithReachability(baselineScan, currentScan);
sw.Stop();
// Log
_output.WriteLine($"Diff with reachability flips: {sw.ElapsedMilliseconds}ms");
_output.WriteLine($"Reachability flips: {diff.ReachabilityFlips.Count}");
_output.WriteLine($"Baseline: {baseline}ms, Threshold (2×): {threshold}ms");
// Assert
sw.ElapsedMilliseconds.Should().BeLessThanOrEqualTo(threshold,
$"Diff with reachability exceeded 2× regression gate ({sw.ElapsedMilliseconds}ms > {threshold}ms)");
}
#endregion
#region Memory Efficiency
[Fact]
public void LargeDiff_MemoryEfficient_Under50MB()
{
// Arrange
const int packageCount = 5000;
const int vulnCount = 1000;
GC.Collect();
GC.WaitForPendingFinalizers();
var beforeMem = GC.GetTotalMemory(true);
// Act
var baselineScan = GenerateScanData(packageCount, vulnCount, seed: 42);
var currentScan = GenerateScanData(packageCount + 200, vulnCount + 100, seed: 43);
var diff = ComputeDiff(baselineScan, currentScan);
var sarif = GenerateSarif(diff);
GC.Collect();
GC.WaitForPendingFinalizers();
var afterMem = GC.GetTotalMemory(true);
var memoryUsedMB = (afterMem - beforeMem) / (1024.0 * 1024.0);
// Log
_output.WriteLine($"Large diff memory usage: {memoryUsedMB:F2}MB");
_output.WriteLine($"SARIF output size: {sarif.Length / 1024.0:F1}KB");
// Assert
memoryUsedMB.Should().BeLessThan(50,
$"Large diff memory usage ({memoryUsedMB:F2}MB) exceeds 50MB threshold");
// Keep objects alive for measurement
(baselineScan.Packages.Count + currentScan.Packages.Count).Should().BeGreaterThan(0);
}
#endregion
#region Test Infrastructure
private static SmartDiffScanData GenerateScanData(int packageCount, int vulnCount, int seed)
{
var random = new Random(seed);
var packages = new List();
var vulnerabilities = new List();
for (int i = 0; i < packageCount; i++)
{
packages.Add(new SmartDiffPackage
{
Name = $"package-{i:D5}",
Version = $"1.{random.Next(0, 10)}.{random.Next(0, 100)}",
Ecosystem = random.Next(0, 3) switch { 0 => "npm", 1 => "nuget", _ => "pypi" }
});
}
for (int i = 0; i < vulnCount; i++)
{
var pkg = packages[random.Next(0, packages.Count)];
vulnerabilities.Add(new SmartDiffVuln
{
CveId = $"CVE-2024-{10000 + i}",
Package = pkg.Name,
Version = pkg.Version,
Severity = random.Next(0, 4) switch { 0 => "LOW", 1 => "MEDIUM", 2 => "HIGH", _ => "CRITICAL" },
IsReachable = random.NextDouble() > 0.5,
ReachabilityTier = random.Next(0, 3) switch { 0 => "imported", 1 => "called", _ => "executed" }
});
}
return new SmartDiffScanData { Packages = packages, Vulnerabilities = vulnerabilities };
}
private static SmartDiffScanData GenerateScanDataWithReachability(
int packageCount, int vulnCount, double reachableRatio, int seed)
{
var data = GenerateScanData(packageCount, vulnCount, seed);
var random = new Random(seed + 1000);
foreach (var vuln in data.Vulnerabilities)
{
vuln.IsReachable = random.NextDouble() < reachableRatio;
}
return data;
}
private static SmartDiffResult ComputeDiff(SmartDiffScanData baseline, SmartDiffScanData current)
{
var baselineSet = baseline.Vulnerabilities
.Select(v => (v.CveId, v.Package, v.Version))
.ToHashSet();
var currentSet = current.Vulnerabilities
.Select(v => (v.CveId, v.Package, v.Version))
.ToHashSet();
return new SmartDiffResult
{
Added = current.Vulnerabilities
.Where(v => !baselineSet.Contains((v.CveId, v.Package, v.Version)))
.ToList(),
Removed = baseline.Vulnerabilities
.Where(v => !currentSet.Contains((v.CveId, v.Package, v.Version)))
.ToList(),
ReachabilityFlips = new List()
};
}
private static SmartDiffResult ComputeDiffWithReachability(SmartDiffScanData baseline, SmartDiffScanData current)
{
var diff = ComputeDiff(baseline, current);
// Find reachability flips (same vuln, different reachability)
var baselineDict = baseline.Vulnerabilities
.ToDictionary(v => (v.CveId, v.Package, v.Version));
diff.ReachabilityFlips = current.Vulnerabilities
.Where(v => baselineDict.TryGetValue((v.CveId, v.Package, v.Version), out var b)
&& b.IsReachable != v.IsReachable)
.ToList();
return diff;
}
private static string GenerateSarif(SmartDiffResult diff)
{
var sarif = new
{
version = "2.1.0",
schema = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
runs = new[]
{
new
{
tool = new { driver = new { name = "StellaOps.SmartDiff", version = "1.0.0" } },
results = diff.Added.Select(v => new
{
ruleId = v.CveId,
message = new { text = $"New vulnerability: {v.CveId} in {v.Package}@{v.Version}" },
level = v.Severity switch { "CRITICAL" => "error", "HIGH" => "error", _ => "warning" },
properties = new { severity = v.Severity, reachable = v.IsReachable }
}).ToArray()
}
}
};
return JsonSerializer.Serialize(sarif, new JsonSerializerOptions { WriteIndented = false });
}
private static SmartDiffVuln CreateFinding(string cveId, string severity, bool reachable, string tier)
{
return new SmartDiffVuln
{
CveId = cveId,
Package = "test-package",
Version = "1.0.0",
Severity = severity,
IsReachable = reachable,
ReachabilityTier = tier
};
}
private static double ScoreFinding(SmartDiffVuln finding)
{
// Simplified scoring algorithm
var baseScore = finding.Severity switch
{
"CRITICAL" => 10.0,
"HIGH" => 7.5,
"MEDIUM" => 5.0,
"LOW" => 2.5,
_ => 1.0
};
var reachabilityMultiplier = finding.IsReachable ? 1.5 : 1.0;
var tierMultiplier = finding.ReachabilityTier switch
{
"executed" => 1.5,
"called" => 1.2,
"imported" => 1.0,
_ => 0.8
};
return baseScore * reachabilityMultiplier * tierMultiplier;
}
private static List ScoreBatch(List findings)
{
return findings.Select(ScoreFinding).ToList();
}
#endregion
#region Test Models
private sealed class SmartDiffScanData
{
public List Packages { get; init; } = new();
public List Vulnerabilities { get; init; } = new();
}
private sealed class SmartDiffPackage
{
public required string Name { get; init; }
public required string Version { get; init; }
public required string Ecosystem { get; init; }
}
private sealed class SmartDiffVuln
{
public required string CveId { get; init; }
public required string Package { get; init; }
public required string Version { get; init; }
public required string Severity { get; set; }
public bool IsReachable { get; set; }
public required string ReachabilityTier { get; set; }
}
private sealed class SmartDiffResult
{
public List Added { get; init; } = new();
public List Removed { get; init; } = new();
public List ReachabilityFlips { get; set; } = new();
}
#endregion
}