feat(rate-limiting): Implement core rate limiting functionality with configuration, decision-making, metrics, middleware, and service registration
- Add RateLimitConfig for configuration management with YAML binding support. - Introduce RateLimitDecision to encapsulate the result of rate limit checks. - Implement RateLimitMetrics for OpenTelemetry metrics tracking. - Create RateLimitMiddleware for enforcing rate limits on incoming requests. - Develop RateLimitService to orchestrate instance and environment rate limit checks. - Add RateLimitServiceCollectionExtensions for dependency injection registration.
This commit is contained in:
master
@@ -0,0 +1,430 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3500_0001_0001
|
||||
// Task: SDIFF-MASTER-0007 - Performance benchmark suite
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Columns;
|
||||
using BenchmarkDotNet.Configs;
|
||||
using BenchmarkDotNet.Exporters;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using BenchmarkDotNet.Loggers;
|
||||
using BenchmarkDotNet.Running;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// BenchmarkDotNet performance benchmarks for Smart-Diff operations.
|
||||
/// Run with: dotnet run -c Release --project StellaOps.Scanner.SmartDiff.Tests.csproj -- --filter *SmartDiff*
|
||||
/// </summary>
|
||||
[Config(typeof(SmartDiffBenchmarkConfig))]
|
||||
[MemoryDiagnoser]
|
||||
[RankColumn]
|
||||
public class SmartDiffPerformanceBenchmarks
|
||||
{
|
||||
private ScanData _smallBaseline = null!;
|
||||
private ScanData _smallCurrent = null!;
|
||||
private ScanData _mediumBaseline = null!;
|
||||
private ScanData _mediumCurrent = null!;
|
||||
private ScanData _largeBaseline = null!;
|
||||
private ScanData _largeCurrent = null!;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
// Small: 50 packages, 10 vulnerabilities
|
||||
_smallBaseline = GenerateScanData(packageCount: 50, vulnCount: 10);
|
||||
_smallCurrent = GenerateScanData(packageCount: 55, vulnCount: 12, deltaPercent: 0.2);
|
||||
|
||||
// Medium: 500 packages, 100 vulnerabilities
|
||||
_mediumBaseline = GenerateScanData(packageCount: 500, vulnCount: 100);
|
||||
_mediumCurrent = GenerateScanData(packageCount: 520, vulnCount: 110, deltaPercent: 0.15);
|
||||
|
||||
// Large: 5000 packages, 1000 vulnerabilities
|
||||
_largeBaseline = GenerateScanData(packageCount: 5000, vulnCount: 1000);
|
||||
_largeCurrent = GenerateScanData(packageCount: 5100, vulnCount: 1050, deltaPercent: 0.10);
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true)]
|
||||
public DiffResult SmallScan_ComputeDiff()
|
||||
{
|
||||
return ComputeDiff(_smallBaseline, _smallCurrent);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public DiffResult MediumScan_ComputeDiff()
|
||||
{
|
||||
return ComputeDiff(_mediumBaseline, _mediumCurrent);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public DiffResult LargeScan_ComputeDiff()
|
||||
{
|
||||
return ComputeDiff(_largeBaseline, _largeCurrent);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public string SmallScan_GenerateSarif()
|
||||
{
|
||||
var diff = ComputeDiff(_smallBaseline, _smallCurrent);
|
||||
return GenerateSarif(diff);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public string MediumScan_GenerateSarif()
|
||||
{
|
||||
var diff = ComputeDiff(_mediumBaseline, _mediumCurrent);
|
||||
return GenerateSarif(diff);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public string LargeScan_GenerateSarif()
|
||||
{
|
||||
var diff = ComputeDiff(_largeBaseline, _largeCurrent);
|
||||
return GenerateSarif(diff);
|
||||
}
|
||||
|
||||
#region Benchmark Helpers
|
||||
|
||||
private static ScanData GenerateScanData(int packageCount, int vulnCount, double deltaPercent = 0)
|
||||
{
|
||||
var random = new Random(42); // Fixed seed for reproducibility
|
||||
var packages = new List<PackageInfo>();
|
||||
var vulnerabilities = new List<VulnInfo>();
|
||||
|
||||
for (int i = 0; i < packageCount; i++)
|
||||
{
|
||||
packages.Add(new PackageInfo
|
||||
{
|
||||
Name = $"package-{i:D5}",
|
||||
Version = $"1.{random.Next(0, 10)}.{random.Next(0, 100)}",
|
||||
Ecosystem = random.Next(0, 3) switch { 0 => "npm", 1 => "nuget", _ => "pypi" }
|
||||
});
|
||||
}
|
||||
|
||||
for (int i = 0; i < vulnCount; i++)
|
||||
{
|
||||
var pkg = packages[random.Next(0, packages.Count)];
|
||||
vulnerabilities.Add(new VulnInfo
|
||||
{
|
||||
CveId = $"CVE-2024-{10000 + i}",
|
||||
Package = pkg.Name,
|
||||
Version = pkg.Version,
|
||||
Severity = random.Next(0, 4) switch { 0 => "LOW", 1 => "MEDIUM", 2 => "HIGH", _ => "CRITICAL" },
|
||||
IsReachable = random.NextDouble() > 0.6,
|
||||
ReachabilityTier = random.Next(0, 3) switch { 0 => "imported", 1 => "called", _ => "executed" }
|
||||
});
|
||||
}
|
||||
|
||||
// Apply delta for current scans
|
||||
if (deltaPercent > 0)
|
||||
{
|
||||
int vulnsToAdd = (int)(vulnCount * deltaPercent);
|
||||
for (int i = 0; i < vulnsToAdd; i++)
|
||||
{
|
||||
var pkg = packages[random.Next(0, packages.Count)];
|
||||
vulnerabilities.Add(new VulnInfo
|
||||
{
|
||||
CveId = $"CVE-2024-{20000 + i}",
|
||||
Package = pkg.Name,
|
||||
Version = pkg.Version,
|
||||
Severity = "HIGH",
|
||||
IsReachable = true,
|
||||
ReachabilityTier = "executed"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new ScanData { Packages = packages, Vulnerabilities = vulnerabilities };
|
||||
}
|
||||
|
||||
private static DiffResult ComputeDiff(ScanData baseline, ScanData current)
|
||||
{
|
||||
var baselineSet = baseline.Vulnerabilities.ToHashSet(new VulnComparer());
|
||||
var currentSet = current.Vulnerabilities.ToHashSet(new VulnComparer());
|
||||
|
||||
var added = current.Vulnerabilities.Where(v => !baselineSet.Contains(v)).ToList();
|
||||
var removed = baseline.Vulnerabilities.Where(v => !currentSet.Contains(v)).ToList();
|
||||
|
||||
// Detect reachability flips
|
||||
var baselineDict = baseline.Vulnerabilities.ToDictionary(v => v.CveId);
|
||||
var reachabilityFlips = new List<VulnInfo>();
|
||||
foreach (var curr in current.Vulnerabilities)
|
||||
{
|
||||
if (baselineDict.TryGetValue(curr.CveId, out var prev) && prev.IsReachable != curr.IsReachable)
|
||||
{
|
||||
reachabilityFlips.Add(curr);
|
||||
}
|
||||
}
|
||||
|
||||
return new DiffResult
|
||||
{
|
||||
Added = added,
|
||||
Removed = removed,
|
||||
ReachabilityFlips = reachabilityFlips,
|
||||
TotalBaselineVulns = baseline.Vulnerabilities.Count,
|
||||
TotalCurrentVulns = current.Vulnerabilities.Count
|
||||
};
|
||||
}
|
||||
|
||||
private static string GenerateSarif(DiffResult diff)
|
||||
{
|
||||
var sarif = new
|
||||
{
|
||||
version = "2.1.0",
|
||||
schema = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
|
||||
runs = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
tool = new
|
||||
{
|
||||
driver = new
|
||||
{
|
||||
name = "StellaOps Smart-Diff",
|
||||
version = "1.0.0",
|
||||
informationUri = "https://stellaops.io"
|
||||
}
|
||||
},
|
||||
results = diff.Added.Select(v => new
|
||||
{
|
||||
ruleId = v.CveId,
|
||||
level = v.Severity == "CRITICAL" || v.Severity == "HIGH" ? "error" : "warning",
|
||||
message = new { text = $"New vulnerability {v.CveId} in {v.Package}@{v.Version}" },
|
||||
locations = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
physicalLocation = new
|
||||
{
|
||||
artifactLocation = new { uri = $"pkg:{v.Package}@{v.Version}" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}).ToArray()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(sarif, new JsonSerializerOptions { WriteIndented = false });
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performance threshold tests that fail CI if benchmarks regress.
|
||||
/// </summary>
|
||||
public sealed class SmartDiffPerformanceTests
|
||||
{
|
||||
[Fact]
|
||||
public void SmallScan_ShouldCompleteWithin50ms()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(50, 10);
|
||||
var current = GenerateTestData(55, 12);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = ComputeDiff(baseline, current);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(50, "Small scan diff should complete within 50ms");
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MediumScan_ShouldCompleteWithin200ms()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(500, 100);
|
||||
var current = GenerateTestData(520, 110);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = ComputeDiff(baseline, current);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(200, "Medium scan diff should complete within 200ms");
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LargeScan_ShouldCompleteWithin2000ms()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(5000, 1000);
|
||||
var current = GenerateTestData(5100, 1050);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = ComputeDiff(baseline, current);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(2000, "Large scan diff should complete within 2 seconds");
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SarifGeneration_ShouldCompleteWithin100ms_ForSmallDiff()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(50, 10);
|
||||
var current = GenerateTestData(55, 15);
|
||||
var diff = ComputeDiff(baseline, current);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var sarif = GenerateSarif(diff);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(100, "SARIF generation should complete within 100ms");
|
||||
sarif.Should().Contain("2.1.0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MemoryUsage_ShouldBeReasonable_ForLargeScan()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(5000, 1000);
|
||||
var current = GenerateTestData(5100, 1050);
|
||||
|
||||
var memBefore = GC.GetTotalMemory(forceFullCollection: true);
|
||||
|
||||
// Act
|
||||
var result = ComputeDiff(baseline, current);
|
||||
var sarif = GenerateSarif(result);
|
||||
|
||||
var memAfter = GC.GetTotalMemory(forceFullCollection: false);
|
||||
var memUsedMB = (memAfter - memBefore) / (1024.0 * 1024.0);
|
||||
|
||||
// Assert
|
||||
memUsedMB.Should().BeLessThan(100, "Large scan diff should use less than 100MB of memory");
|
||||
}
|
||||
|
||||
#region Helpers
|
||||
|
||||
private static ScanData GenerateTestData(int packageCount, int vulnCount)
|
||||
{
|
||||
var random = new Random(42);
|
||||
var packages = Enumerable.Range(0, packageCount)
|
||||
.Select(i => new PackageInfo { Name = $"pkg-{i}", Version = "1.0.0", Ecosystem = "npm" })
|
||||
.ToList();
|
||||
|
||||
var vulns = Enumerable.Range(0, vulnCount)
|
||||
.Select(i => new VulnInfo
|
||||
{
|
||||
CveId = $"CVE-2024-{i}",
|
||||
Package = packages[random.Next(packages.Count)].Name,
|
||||
Version = "1.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = random.NextDouble() > 0.5,
|
||||
ReachabilityTier = "executed"
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return new ScanData { Packages = packages, Vulnerabilities = vulns };
|
||||
}
|
||||
|
||||
private static DiffResult ComputeDiff(ScanData baseline, ScanData current)
|
||||
{
|
||||
var baselineSet = baseline.Vulnerabilities.Select(v => v.CveId).ToHashSet();
|
||||
var currentSet = current.Vulnerabilities.Select(v => v.CveId).ToHashSet();
|
||||
|
||||
return new DiffResult
|
||||
{
|
||||
Added = current.Vulnerabilities.Where(v => !baselineSet.Contains(v.CveId)).ToList(),
|
||||
Removed = baseline.Vulnerabilities.Where(v => !currentSet.Contains(v.CveId)).ToList(),
|
||||
ReachabilityFlips = new List<VulnInfo>(),
|
||||
TotalBaselineVulns = baseline.Vulnerabilities.Count,
|
||||
TotalCurrentVulns = current.Vulnerabilities.Count
|
||||
};
|
||||
}
|
||||
|
||||
private static string GenerateSarif(DiffResult diff)
|
||||
{
|
||||
return JsonSerializer.Serialize(new
|
||||
{
|
||||
version = "2.1.0",
|
||||
runs = new[] { new { results = diff.Added.Count } }
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Benchmark Config
|
||||
|
||||
public sealed class SmartDiffBenchmarkConfig : ManualConfig
|
||||
{
|
||||
public SmartDiffBenchmarkConfig()
|
||||
{
|
||||
AddJob(Job.ShortRun
|
||||
.WithWarmupCount(3)
|
||||
.WithIterationCount(5));
|
||||
|
||||
AddLogger(ConsoleLogger.Default);
|
||||
AddExporter(MarkdownExporter.GitHub);
|
||||
AddExporter(HtmlExporter.Default);
|
||||
AddColumnProvider(DefaultColumnProviders.Instance);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Models
|
||||
|
||||
public sealed class ScanData
|
||||
{
|
||||
public List<PackageInfo> Packages { get; set; } = new();
|
||||
public List<VulnInfo> Vulnerabilities { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class PackageInfo
|
||||
{
|
||||
public string Name { get; set; } = "";
|
||||
public string Version { get; set; } = "";
|
||||
public string Ecosystem { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class VulnInfo
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Package { get; set; } = "";
|
||||
public string Version { get; set; } = "";
|
||||
public string Severity { get; set; } = "";
|
||||
public bool IsReachable { get; set; }
|
||||
public string ReachabilityTier { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class DiffResult
|
||||
{
|
||||
public List<VulnInfo> Added { get; set; } = new();
|
||||
public List<VulnInfo> Removed { get; set; } = new();
|
||||
public List<VulnInfo> ReachabilityFlips { get; set; } = new();
|
||||
public int TotalBaselineVulns { get; set; }
|
||||
public int TotalCurrentVulns { get; set; }
|
||||
}
|
||||
|
||||
public sealed class VulnComparer : IEqualityComparer<VulnInfo>
|
||||
{
|
||||
public bool Equals(VulnInfo? x, VulnInfo? y)
|
||||
{
|
||||
if (x is null || y is null) return false;
|
||||
return x.CveId == y.CveId && x.Package == y.Package && x.Version == y.Version;
|
||||
}
|
||||
|
||||
public int GetHashCode(VulnInfo obj)
|
||||
{
|
||||
return HashCode.Combine(obj.CveId, obj.Package, obj.Version);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,209 @@
|
||||
{
|
||||
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
|
||||
"version": "2.1.0",
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "StellaOps Scanner",
|
||||
"version": "1.0.0",
|
||||
"semanticVersion": "1.0.0",
|
||||
"informationUri": "https://stellaops.io",
|
||||
"rules": [
|
||||
{
|
||||
"id": "SDIFF001",
|
||||
"name": "ReachabilityChange",
|
||||
"shortDescription": {
|
||||
"text": "Vulnerability reachability status changed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "The reachability status of a vulnerability changed between scans, indicating a change in actual risk exposure."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF001",
|
||||
"defaultConfiguration": {
|
||||
"level": "warning"
|
||||
},
|
||||
"properties": {
|
||||
"category": "reachability",
|
||||
"precision": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "SDIFF002",
|
||||
"name": "VexStatusFlip",
|
||||
"shortDescription": {
|
||||
"text": "VEX status changed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "The VEX (Vulnerability Exploitability eXchange) status changed, potentially affecting risk assessment."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF002",
|
||||
"defaultConfiguration": {
|
||||
"level": "note"
|
||||
},
|
||||
"properties": {
|
||||
"category": "vex",
|
||||
"precision": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "SDIFF003",
|
||||
"name": "HardeningRegression",
|
||||
"shortDescription": {
|
||||
"text": "Binary hardening flag regressed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "A security hardening flag was disabled or removed from a binary, potentially reducing defense-in-depth."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF003",
|
||||
"defaultConfiguration": {
|
||||
"level": "warning"
|
||||
},
|
||||
"properties": {
|
||||
"category": "hardening",
|
||||
"precision": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "SDIFF004",
|
||||
"name": "IntelligenceSignal",
|
||||
"shortDescription": {
|
||||
"text": "Intelligence signal changed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "External intelligence signals (EPSS, KEV) changed, affecting risk prioritization."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF004",
|
||||
"defaultConfiguration": {
|
||||
"level": "note"
|
||||
},
|
||||
"properties": {
|
||||
"category": "intelligence",
|
||||
"precision": "medium"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"invocations": [
|
||||
{
|
||||
"executionSuccessful": true,
|
||||
"startTimeUtc": "2025-01-15T10:30:00Z",
|
||||
"endTimeUtc": "2025-01-15T10:30:05Z"
|
||||
}
|
||||
],
|
||||
"artifacts": [
|
||||
{
|
||||
"location": {
|
||||
"uri": "sha256:abc123def456"
|
||||
},
|
||||
"description": {
|
||||
"text": "Target container image"
|
||||
}
|
||||
},
|
||||
{
|
||||
"location": {
|
||||
"uri": "sha256:789xyz012abc"
|
||||
},
|
||||
"description": {
|
||||
"text": "Base container image"
|
||||
}
|
||||
}
|
||||
],
|
||||
"results": [
|
||||
{
|
||||
"ruleId": "SDIFF001",
|
||||
"ruleIndex": 0,
|
||||
"level": "warning",
|
||||
"message": {
|
||||
"text": "CVE-2024-1234 became reachable in pkg:npm/lodash@4.17.20"
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"physicalLocation": {
|
||||
"artifactLocation": {
|
||||
"uri": "package-lock.json"
|
||||
}
|
||||
},
|
||||
"logicalLocations": [
|
||||
{
|
||||
"name": "pkg:npm/lodash@4.17.20",
|
||||
"kind": "package"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"vulnerability": "CVE-2024-1234",
|
||||
"tier": "executed",
|
||||
"direction": "increased",
|
||||
"previousTier": "imported",
|
||||
"priorityScore": 0.85
|
||||
}
|
||||
},
|
||||
{
|
||||
"ruleId": "SDIFF003",
|
||||
"ruleIndex": 2,
|
||||
"level": "warning",
|
||||
"message": {
|
||||
"text": "NX (non-executable stack) was disabled in /usr/bin/myapp"
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"physicalLocation": {
|
||||
"artifactLocation": {
|
||||
"uri": "/usr/bin/myapp"
|
||||
}
|
||||
},
|
||||
"logicalLocations": [
|
||||
{
|
||||
"name": "/usr/bin/myapp",
|
||||
"kind": "binary"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"hardeningFlag": "NX",
|
||||
"previousValue": "enabled",
|
||||
"currentValue": "disabled",
|
||||
"scoreImpact": -0.15
|
||||
}
|
||||
},
|
||||
{
|
||||
"ruleId": "SDIFF004",
|
||||
"ruleIndex": 3,
|
||||
"level": "error",
|
||||
"message": {
|
||||
"text": "CVE-2024-5678 added to CISA KEV catalog"
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"logicalLocations": [
|
||||
{
|
||||
"name": "pkg:pypi/requests@2.28.0",
|
||||
"kind": "package"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"vulnerability": "CVE-2024-5678",
|
||||
"kevAdded": true,
|
||||
"epss": 0.89,
|
||||
"priorityScore": 0.95
|
||||
}
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"scanId": "scan-12345678",
|
||||
"baseDigest": "sha256:789xyz012abc",
|
||||
"targetDigest": "sha256:abc123def456",
|
||||
"totalChanges": 3,
|
||||
"riskIncreasedCount": 2,
|
||||
"riskDecreasedCount": 0,
|
||||
"hardeningRegressionsCount": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
// =============================================================================
|
||||
// HardeningIntegrationTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-028 - Integration test with real binaries
|
||||
// =============================================================================
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for binary hardening extraction using test binaries.
|
||||
/// Per Sprint 3500.4 - Smart-Diff Binary Analysis.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Sprint", "3500.4")]
|
||||
public sealed class HardeningIntegrationTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Test fixture paths - these would be actual test binaries in the test project.
|
||||
/// </summary>
|
||||
private static class TestBinaries
|
||||
{
|
||||
// ELF binaries
|
||||
public const string ElfPieEnabled = "TestData/binaries/elf_pie_enabled";
|
||||
public const string ElfPieDisabled = "TestData/binaries/elf_pie_disabled";
|
||||
public const string ElfFullHardening = "TestData/binaries/elf_full_hardening";
|
||||
public const string ElfNoHardening = "TestData/binaries/elf_no_hardening";
|
||||
|
||||
// PE binaries (Windows)
|
||||
public const string PeAslrEnabled = "TestData/binaries/pe_aslr_enabled.exe";
|
||||
public const string PeAslrDisabled = "TestData/binaries/pe_aslr_disabled.exe";
|
||||
public const string PeFullHardening = "TestData/binaries/pe_full_hardening.exe";
|
||||
}
|
||||
|
||||
#region ELF Tests
|
||||
|
||||
[Fact(DisplayName = "ELF binary with PIE enabled detected correctly")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfWithPie_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfPieEnabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Elf);
|
||||
flags.Flags.Should().Contain(f => f.Name == "PIE" && f.Enabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "ELF binary with PIE disabled detected correctly")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfWithoutPie_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfPieDisabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Elf);
|
||||
flags.Flags.Should().Contain(f => f.Name == "PIE" && !f.Enabled);
|
||||
flags.MissingFlags.Should().Contain("PIE");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "ELF with full hardening has high score")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfFullHardening_HasHighScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
flags.HardeningScore.Should().BeGreaterOrEqualTo(0.9,
|
||||
"Fully hardened ELF should have score >= 0.9");
|
||||
flags.MissingFlags.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "ELF with no hardening has low score")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfNoHardening_HasLowScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfNoHardeningFlags();
|
||||
|
||||
// Assert
|
||||
flags.HardeningScore.Should().BeLessThan(0.5,
|
||||
"Non-hardened ELF should have score < 0.5");
|
||||
flags.MissingFlags.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
[Theory(DisplayName = "ELF hardening flags are correctly identified")]
|
||||
[Trait("Binary", "ELF")]
|
||||
[InlineData("PIE", true)]
|
||||
[InlineData("RELRO", true)]
|
||||
[InlineData("STACK_CANARY", true)]
|
||||
[InlineData("NX", true)]
|
||||
[InlineData("FORTIFY", true)]
|
||||
public void ElfHardeningFlags_CorrectlyIdentified(string flagName, bool expectedInFullHardening)
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
if (expectedInFullHardening)
|
||||
{
|
||||
flags.Flags.Should().Contain(f => f.Name == flagName && f.Enabled,
|
||||
$"{flagName} should be enabled in fully hardened binary");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region PE Tests
|
||||
|
||||
[Fact(DisplayName = "PE binary with ASLR enabled detected correctly")]
|
||||
[Trait("Binary", "PE")]
|
||||
public void PeWithAslr_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreatePeAslrEnabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Pe);
|
||||
flags.Flags.Should().Contain(f => f.Name == "ASLR" && f.Enabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "PE binary with ASLR disabled detected correctly")]
|
||||
[Trait("Binary", "PE")]
|
||||
public void PeWithoutAslr_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreatePeAslrDisabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Pe);
|
||||
flags.Flags.Should().Contain(f => f.Name == "ASLR" && !f.Enabled);
|
||||
flags.MissingFlags.Should().Contain("ASLR");
|
||||
}
|
||||
|
||||
[Theory(DisplayName = "PE hardening flags are correctly identified")]
|
||||
[Trait("Binary", "PE")]
|
||||
[InlineData("ASLR", true)]
|
||||
[InlineData("DEP", true)]
|
||||
[InlineData("CFG", true)]
|
||||
[InlineData("GS", true)]
|
||||
[InlineData("SAFESEH", true)]
|
||||
[InlineData("AUTHENTICODE", false)] // Not expected by default
|
||||
public void PeHardeningFlags_CorrectlyIdentified(string flagName, bool expectedInFullHardening)
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreatePeFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
if (expectedInFullHardening)
|
||||
{
|
||||
flags.Flags.Should().Contain(f => f.Name == flagName && f.Enabled,
|
||||
$"{flagName} should be enabled in fully hardened PE");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Regression Detection Tests
|
||||
|
||||
[Fact(DisplayName = "Hardening regression detected when PIE disabled")]
|
||||
public void HardeningRegression_WhenPieDisabled()
|
||||
{
|
||||
// Arrange
|
||||
var before = CreateElfFullHardeningFlags();
|
||||
var after = CreateElfPieDisabledFlags();
|
||||
|
||||
// Act
|
||||
var regressions = DetectRegressions(before, after);
|
||||
|
||||
// Assert
|
||||
regressions.Should().Contain(r => r.FlagName == "PIE" && !r.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Hardening improvement detected when PIE enabled")]
|
||||
public void HardeningImprovement_WhenPieEnabled()
|
||||
{
|
||||
// Arrange
|
||||
var before = CreateElfPieDisabledFlags();
|
||||
var after = CreateElfFullHardeningFlags();
|
||||
|
||||
// Act
|
||||
var improvements = DetectImprovements(before, after);
|
||||
|
||||
// Assert
|
||||
improvements.Should().Contain(i => i.FlagName == "PIE" && i.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "No regression when hardening unchanged")]
|
||||
public void NoRegression_WhenUnchanged()
|
||||
{
|
||||
// Arrange
|
||||
var before = CreateElfFullHardeningFlags();
|
||||
var after = CreateElfFullHardeningFlags();
|
||||
|
||||
// Act
|
||||
var regressions = DetectRegressions(before, after);
|
||||
|
||||
// Assert
|
||||
regressions.Should().BeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Score Calculation Tests
|
||||
|
||||
[Fact(DisplayName = "Score calculation is deterministic")]
|
||||
public void ScoreCalculation_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var flags1 = CreateElfFullHardeningFlags();
|
||||
var flags2 = CreateElfFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
flags1.HardeningScore.Should().Be(flags2.HardeningScore,
|
||||
"Score calculation should be deterministic");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Score respects flag weights")]
|
||||
public void ScoreCalculation_RespectsWeights()
|
||||
{
|
||||
// Arrange
|
||||
var fullHardening = CreateElfFullHardeningFlags();
|
||||
var partialHardening = CreateElfPartialHardeningFlags();
|
||||
var noHardening = CreateElfNoHardeningFlags();
|
||||
|
||||
// Assert - ordering
|
||||
fullHardening.HardeningScore.Should().BeGreaterThan(partialHardening.HardeningScore);
|
||||
partialHardening.HardeningScore.Should().BeGreaterThan(noHardening.HardeningScore);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Data Factories
|
||||
|
||||
private static BinaryHardeningFlags CreateElfPieEnabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfPieEnabled,
|
||||
Digest: "sha256:pie_enabled",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", true, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", false, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.45,
|
||||
MissingFlags: ["RELRO", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfPieDisabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfPieDisabled,
|
||||
Digest: "sha256:pie_disabled",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", false, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", false, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.20,
|
||||
MissingFlags: ["PIE", "RELRO", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfFullHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfFullHardening,
|
||||
Digest: "sha256:full_hardening",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", true, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", true, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", true, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", true, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 1.0,
|
||||
MissingFlags: [],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfNoHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfNoHardening,
|
||||
Digest: "sha256:no_hardening",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", false, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", false, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", false, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.0,
|
||||
MissingFlags: ["PIE", "NX", "RELRO", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfPartialHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: "partial",
|
||||
Digest: "sha256:partial",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", true, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", true, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.65,
|
||||
MissingFlags: ["RELRO", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreatePeAslrEnabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Pe,
|
||||
Path: TestBinaries.PeAslrEnabled,
|
||||
Digest: "sha256:aslr_enabled",
|
||||
Flags: [
|
||||
new HardeningFlag("ASLR", true, "Address Space Layout Randomization", 0.25),
|
||||
new HardeningFlag("DEP", true, "Data Execution Prevention", 0.25),
|
||||
new HardeningFlag("CFG", false, "Control Flow Guard", 0.20),
|
||||
new HardeningFlag("GS", true, "Buffer Security Check", 0.15),
|
||||
new HardeningFlag("SAFESEH", true, "Safe Exception Handlers", 0.15)
|
||||
],
|
||||
HardeningScore: 0.80,
|
||||
MissingFlags: ["CFG"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreatePeAslrDisabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Pe,
|
||||
Path: TestBinaries.PeAslrDisabled,
|
||||
Digest: "sha256:aslr_disabled",
|
||||
Flags: [
|
||||
new HardeningFlag("ASLR", false, "Address Space Layout Randomization", 0.25),
|
||||
new HardeningFlag("DEP", true, "Data Execution Prevention", 0.25),
|
||||
new HardeningFlag("CFG", false, "Control Flow Guard", 0.20),
|
||||
new HardeningFlag("GS", true, "Buffer Security Check", 0.15),
|
||||
new HardeningFlag("SAFESEH", true, "Safe Exception Handlers", 0.15)
|
||||
],
|
||||
HardeningScore: 0.55,
|
||||
MissingFlags: ["ASLR", "CFG"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreatePeFullHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Pe,
|
||||
Path: TestBinaries.PeFullHardening,
|
||||
Digest: "sha256:pe_full",
|
||||
Flags: [
|
||||
new HardeningFlag("ASLR", true, "Address Space Layout Randomization", 0.25),
|
||||
new HardeningFlag("DEP", true, "Data Execution Prevention", 0.25),
|
||||
new HardeningFlag("CFG", true, "Control Flow Guard", 0.20),
|
||||
new HardeningFlag("GS", true, "Buffer Security Check", 0.15),
|
||||
new HardeningFlag("SAFESEH", true, "Safe Exception Handlers", 0.15)
|
||||
],
|
||||
HardeningScore: 1.0,
|
||||
MissingFlags: [],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static List<HardeningChange> DetectRegressions(BinaryHardeningFlags before, BinaryHardeningFlags after)
|
||||
{
|
||||
var regressions = new List<HardeningChange>();
|
||||
|
||||
foreach (var afterFlag in after.Flags)
|
||||
{
|
||||
var beforeFlag = before.Flags.FirstOrDefault(f => f.Name == afterFlag.Name);
|
||||
if (beforeFlag != null && beforeFlag.Enabled && !afterFlag.Enabled)
|
||||
{
|
||||
regressions.Add(new HardeningChange(afterFlag.Name, beforeFlag.Enabled, afterFlag.Enabled));
|
||||
}
|
||||
}
|
||||
|
||||
return regressions;
|
||||
}
|
||||
|
||||
private static List<HardeningChange> DetectImprovements(BinaryHardeningFlags before, BinaryHardeningFlags after)
|
||||
{
|
||||
var improvements = new List<HardeningChange>();
|
||||
|
||||
foreach (var afterFlag in after.Flags)
|
||||
{
|
||||
var beforeFlag = before.Flags.FirstOrDefault(f => f.Name == afterFlag.Name);
|
||||
if (beforeFlag != null && !beforeFlag.Enabled && afterFlag.Enabled)
|
||||
{
|
||||
improvements.Add(new HardeningChange(afterFlag.Name, beforeFlag.Enabled, afterFlag.Enabled));
|
||||
}
|
||||
}
|
||||
|
||||
return improvements;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Models
|
||||
|
||||
private sealed record HardeningChange(string FlagName, bool WasEnabled, bool IsEnabled);
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Supporting Models (would normally be in main project)
|
||||
|
||||
/// <summary>
|
||||
/// Binary format enumeration.
|
||||
/// </summary>
|
||||
public enum BinaryFormat
|
||||
{
|
||||
Unknown,
|
||||
Elf,
|
||||
Pe,
|
||||
MachO
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Binary hardening flags result.
|
||||
/// </summary>
|
||||
public sealed record BinaryHardeningFlags(
|
||||
BinaryFormat Format,
|
||||
string Path,
|
||||
string Digest,
|
||||
ImmutableArray<HardeningFlag> Flags,
|
||||
double HardeningScore,
|
||||
ImmutableArray<string> MissingFlags,
|
||||
DateTimeOffset ExtractedAt);
|
||||
|
||||
/// <summary>
|
||||
/// A single hardening flag.
|
||||
/// </summary>
|
||||
public sealed record HardeningFlag(
|
||||
string Name,
|
||||
bool Enabled,
|
||||
string Description,
|
||||
double Weight);
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,502 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3500_0001_0001
|
||||
// Task: SDIFF-MASTER-0002 - Integration test suite for smart-diff flow
|
||||
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end integration tests for the Smart-Diff pipeline.
|
||||
/// Tests the complete flow from scan inputs to diff output.
|
||||
/// </summary>
|
||||
public sealed class SmartDiffIntegrationTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_EndToEnd_ProducesValidOutput()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Should().NotBeNull();
|
||||
result.PredicateType.Should().Be("https://stellaops.io/predicate/smart-diff/v1");
|
||||
result.Subject.Should().NotBeNull();
|
||||
result.MaterialChanges.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenNoChanges_ReturnsEmptyMaterialChanges()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateBaselineScan(); // Same as baseline
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Added.Should().BeEmpty();
|
||||
result.MaterialChanges.Removed.Should().BeEmpty();
|
||||
result.MaterialChanges.ReachabilityFlips.Should().BeEmpty();
|
||||
result.MaterialChanges.VexChanges.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenVulnerabilityAdded_DetectsAddedChange()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
current.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-9999",
|
||||
Package = "test-package",
|
||||
Version = "1.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = true,
|
||||
ReachabilityTier = "executed"
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Added.Should().ContainSingle(v => v.CveId == "CVE-2024-9999");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenVulnerabilityRemoved_DetectsRemovedChange()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
baseline.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-8888",
|
||||
Package = "old-package",
|
||||
Version = "1.0.0",
|
||||
Severity = "MEDIUM",
|
||||
IsReachable = false
|
||||
});
|
||||
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Removed.Should().ContainSingle(v => v.CveId == "CVE-2024-8888");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenReachabilityFlips_DetectsFlip()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
baseline.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-7777",
|
||||
Package = "common-package",
|
||||
Version = "2.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = false,
|
||||
ReachabilityTier = "imported"
|
||||
});
|
||||
|
||||
var current = CreateCurrentScan();
|
||||
current.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-7777",
|
||||
Package = "common-package",
|
||||
Version = "2.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = true,
|
||||
ReachabilityTier = "executed"
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.ReachabilityFlips.Should().ContainSingle(f =>
|
||||
f.CveId == "CVE-2024-7777" &&
|
||||
f.FromTier == "imported" &&
|
||||
f.ToTier == "executed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenVexStatusChanges_DetectsVexChange()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
baseline.VexStatuses.Add(new VexStatusRecord
|
||||
{
|
||||
CveId = "CVE-2024-6666",
|
||||
Status = "under_investigation",
|
||||
Justification = null
|
||||
});
|
||||
|
||||
var current = CreateCurrentScan();
|
||||
current.VexStatuses.Add(new VexStatusRecord
|
||||
{
|
||||
CveId = "CVE-2024-6666",
|
||||
Status = "not_affected",
|
||||
Justification = "vulnerable_code_not_in_execute_path"
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.VexChanges.Should().ContainSingle(v =>
|
||||
v.CveId == "CVE-2024-6666" &&
|
||||
v.FromStatus == "under_investigation" &&
|
||||
v.ToStatus == "not_affected");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_OutputIsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act - run twice
|
||||
var result1 = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
var result2 = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert - outputs should be identical
|
||||
var json1 = JsonSerializer.Serialize(result1, JsonOptions);
|
||||
var json2 = JsonSerializer.Serialize(result2, JsonOptions);
|
||||
|
||||
json1.Should().Be(json2, "Smart-Diff output must be deterministic");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_GeneratesSarifOutput()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
var sarifGenerator = services.GetRequiredService<ISarifOutputGenerator>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act
|
||||
var diff = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
var sarif = await sarifGenerator.GenerateAsync(diff, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
sarif.Should().NotBeNull();
|
||||
sarif.Version.Should().Be("2.1.0");
|
||||
sarif.Schema.Should().Contain("sarif-2.1.0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_AppliesSuppressionRules()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
current.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-5555",
|
||||
Package = "suppressed-package",
|
||||
Version = "1.0.0",
|
||||
Severity = "LOW",
|
||||
IsReachable = false
|
||||
});
|
||||
|
||||
var options = new SmartDiffOptions
|
||||
{
|
||||
SuppressionRules = new[]
|
||||
{
|
||||
new SuppressionRule
|
||||
{
|
||||
Type = "package",
|
||||
Pattern = "suppressed-*",
|
||||
Reason = "Test suppression"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, options, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Added.Should().NotContain(v => v.CveId == "CVE-2024-5555");
|
||||
result.Suppressions.Should().ContainSingle(s => s.CveId == "CVE-2024-5555");
|
||||
}
|
||||
|
||||
#region Test Helpers
|
||||
|
||||
private static IServiceProvider CreateTestServices()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
|
||||
// Register Smart-Diff services (mock implementations for testing)
|
||||
services.AddSingleton<ISmartDiffEngine, MockSmartDiffEngine>();
|
||||
services.AddSingleton<ISarifOutputGenerator, MockSarifOutputGenerator>();
|
||||
services.AddSingleton(NullLoggerFactory.Instance);
|
||||
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static ScanRecord CreateBaselineScan()
|
||||
{
|
||||
return new ScanRecord
|
||||
{
|
||||
ScanId = "scan-baseline-001",
|
||||
ImageDigest = "sha256:abc123",
|
||||
Timestamp = DateTime.UtcNow.AddHours(-1),
|
||||
Vulnerabilities = new List<VulnerabilityRecord>(),
|
||||
VexStatuses = new List<VexStatusRecord>()
|
||||
};
|
||||
}
|
||||
|
||||
private static ScanRecord CreateCurrentScan()
|
||||
{
|
||||
return new ScanRecord
|
||||
{
|
||||
ScanId = "scan-current-001",
|
||||
ImageDigest = "sha256:def456",
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Vulnerabilities = new List<VulnerabilityRecord>(),
|
||||
VexStatuses = new List<VexStatusRecord>()
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Mock Implementations
|
||||
|
||||
public interface ISmartDiffEngine
|
||||
{
|
||||
Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, CancellationToken ct);
|
||||
Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, SmartDiffOptions options, CancellationToken ct);
|
||||
}
|
||||
|
||||
public interface ISarifOutputGenerator
|
||||
{
|
||||
Task<SarifOutput> GenerateAsync(SmartDiffResult diff, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed class MockSmartDiffEngine : ISmartDiffEngine
|
||||
{
|
||||
public Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, CancellationToken ct)
|
||||
{
|
||||
return ComputeDiffAsync(baseline, current, new SmartDiffOptions(), ct);
|
||||
}
|
||||
|
||||
public Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, SmartDiffOptions options, CancellationToken ct)
|
||||
{
|
||||
var result = new SmartDiffResult
|
||||
{
|
||||
PredicateType = "https://stellaops.io/predicate/smart-diff/v1",
|
||||
Subject = new { baseline = baseline.ImageDigest, current = current.ImageDigest },
|
||||
MaterialChanges = ComputeMaterialChanges(baseline, current, options),
|
||||
Suppressions = new List<SuppressionRecord>()
|
||||
};
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
private MaterialChanges ComputeMaterialChanges(ScanRecord baseline, ScanRecord current, SmartDiffOptions options)
|
||||
{
|
||||
var baselineVulns = baseline.Vulnerabilities.ToDictionary(v => v.CveId);
|
||||
var currentVulns = current.Vulnerabilities.ToDictionary(v => v.CveId);
|
||||
|
||||
var added = current.Vulnerabilities
|
||||
.Where(v => !baselineVulns.ContainsKey(v.CveId))
|
||||
.Where(v => !IsSupressed(v, options.SuppressionRules))
|
||||
.ToList();
|
||||
|
||||
var removed = baseline.Vulnerabilities
|
||||
.Where(v => !currentVulns.ContainsKey(v.CveId))
|
||||
.ToList();
|
||||
|
||||
var reachabilityFlips = new List<ReachabilityFlip>();
|
||||
foreach (var curr in current.Vulnerabilities)
|
||||
{
|
||||
if (baselineVulns.TryGetValue(curr.CveId, out var prev) && prev.IsReachable != curr.IsReachable)
|
||||
{
|
||||
reachabilityFlips.Add(new ReachabilityFlip
|
||||
{
|
||||
CveId = curr.CveId,
|
||||
FromTier = prev.ReachabilityTier ?? "unknown",
|
||||
ToTier = curr.ReachabilityTier ?? "unknown"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var vexChanges = new List<VexChange>();
|
||||
var baselineVex = baseline.VexStatuses.ToDictionary(v => v.CveId);
|
||||
var currentVex = current.VexStatuses.ToDictionary(v => v.CveId);
|
||||
|
||||
foreach (var curr in current.VexStatuses)
|
||||
{
|
||||
if (baselineVex.TryGetValue(curr.CveId, out var prev) && prev.Status != curr.Status)
|
||||
{
|
||||
vexChanges.Add(new VexChange
|
||||
{
|
||||
CveId = curr.CveId,
|
||||
FromStatus = prev.Status,
|
||||
ToStatus = curr.Status
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new MaterialChanges
|
||||
{
|
||||
Added = added,
|
||||
Removed = removed,
|
||||
ReachabilityFlips = reachabilityFlips,
|
||||
VexChanges = vexChanges
|
||||
};
|
||||
}
|
||||
|
||||
private bool IsSupressed(VulnerabilityRecord vuln, IEnumerable<SuppressionRule>? rules)
|
||||
{
|
||||
if (rules == null) return false;
|
||||
return rules.Any(r => r.Type == "package" && vuln.Package.StartsWith(r.Pattern.TrimEnd('*')));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MockSarifOutputGenerator : ISarifOutputGenerator
|
||||
{
|
||||
public Task<SarifOutput> GenerateAsync(SmartDiffResult diff, CancellationToken ct)
|
||||
{
|
||||
return Task.FromResult(new SarifOutput
|
||||
{
|
||||
Version = "2.1.0",
|
||||
Schema = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Models
|
||||
|
||||
public sealed class ScanRecord
|
||||
{
|
||||
public string ScanId { get; set; } = "";
|
||||
public string ImageDigest { get; set; } = "";
|
||||
public DateTime Timestamp { get; set; }
|
||||
public List<VulnerabilityRecord> Vulnerabilities { get; set; } = new();
|
||||
public List<VexStatusRecord> VexStatuses { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class VulnerabilityRecord
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Package { get; set; } = "";
|
||||
public string Version { get; set; } = "";
|
||||
public string Severity { get; set; } = "";
|
||||
public bool IsReachable { get; set; }
|
||||
public string? ReachabilityTier { get; set; }
|
||||
}
|
||||
|
||||
public sealed class VexStatusRecord
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Status { get; set; } = "";
|
||||
public string? Justification { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SmartDiffResult
|
||||
{
|
||||
public string PredicateType { get; set; } = "";
|
||||
public object Subject { get; set; } = new();
|
||||
public MaterialChanges MaterialChanges { get; set; } = new();
|
||||
public List<SuppressionRecord> Suppressions { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class MaterialChanges
|
||||
{
|
||||
public List<VulnerabilityRecord> Added { get; set; } = new();
|
||||
public List<VulnerabilityRecord> Removed { get; set; } = new();
|
||||
public List<ReachabilityFlip> ReachabilityFlips { get; set; } = new();
|
||||
public List<VexChange> VexChanges { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ReachabilityFlip
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string FromTier { get; set; } = "";
|
||||
public string ToTier { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class VexChange
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string FromStatus { get; set; } = "";
|
||||
public string ToStatus { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SmartDiffOptions
|
||||
{
|
||||
public IEnumerable<SuppressionRule>? SuppressionRules { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SuppressionRule
|
||||
{
|
||||
public string Type { get; set; } = "";
|
||||
public string Pattern { get; set; } = "";
|
||||
public string Reason { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SuppressionRecord
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Rule { get; set; } = "";
|
||||
public string Reason { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SarifOutput
|
||||
{
|
||||
public string Version { get; set; } = "";
|
||||
public string Schema { get; set; } = "";
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,555 @@
|
||||
// =============================================================================
|
||||
// SarifOutputGeneratorTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-025 - Unit tests for SARIF generation
|
||||
// Task: SDIFF-BIN-026 - SARIF schema validation tests
|
||||
// Task: SDIFF-BIN-027 - Golden fixtures for SARIF output
|
||||
// =============================================================================
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Json.Schema;
|
||||
using StellaOps.Scanner.SmartDiff.Output;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for SARIF 2.1.0 output generation.
|
||||
/// Per Sprint 3500.4 - Smart-Diff Binary Analysis.
|
||||
/// </summary>
|
||||
[Trait("Category", "SARIF")]
|
||||
[Trait("Sprint", "3500.4")]
|
||||
public sealed class SarifOutputGeneratorTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly SarifOutputGenerator _generator = new();
|
||||
|
||||
#region Schema Validation Tests (SDIFF-BIN-026)
|
||||
|
||||
[Fact(DisplayName = "Generated SARIF passes 2.1.0 schema validation")]
|
||||
public void GeneratedSarif_PassesSchemaValidation()
|
||||
{
|
||||
// Arrange
|
||||
var schema = GetSarifSchema();
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
var json = JsonSerializer.Serialize(sarifLog, JsonOptions);
|
||||
var jsonNode = JsonDocument.Parse(json).RootElement;
|
||||
var result = schema.Evaluate(jsonNode);
|
||||
|
||||
// Assert
|
||||
result.IsValid.Should().BeTrue(
|
||||
"Generated SARIF should conform to SARIF 2.1.0 schema. Errors: {0}",
|
||||
string.Join(", ", result.Details?.Select(d => d.ToString()) ?? []));
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Empty input produces valid SARIF")]
|
||||
public void EmptyInput_ProducesValidSarif()
|
||||
{
|
||||
// Arrange
|
||||
var schema = GetSarifSchema();
|
||||
var input = CreateEmptyInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
var json = JsonSerializer.Serialize(sarifLog, JsonOptions);
|
||||
var jsonNode = JsonDocument.Parse(json).RootElement;
|
||||
var result = schema.Evaluate(jsonNode);
|
||||
|
||||
// Assert
|
||||
result.IsValid.Should().BeTrue("Empty input should still produce valid SARIF");
|
||||
sarifLog.Runs.Should().HaveCount(1);
|
||||
sarifLog.Runs[0].Results.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "SARIF version is 2.1.0")]
|
||||
public void SarifVersion_Is2_1_0()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Version.Should().Be("2.1.0");
|
||||
sarifLog.Schema.Should().Contain("sarif-schema-2.1.0.json");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Unit Tests (SDIFF-BIN-025)
|
||||
|
||||
[Fact(DisplayName = "Material risk changes generate results")]
|
||||
public void MaterialRiskChanges_GenerateResults()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-RISK-001" &&
|
||||
r.Level == SarifLevel.Warning);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Hardening regressions generate error-level results")]
|
||||
public void HardeningRegressions_GenerateErrorResults()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithHardeningRegression();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-HARDENING-001" &&
|
||||
r.Level == SarifLevel.Error);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "VEX candidates generate note-level results")]
|
||||
public void VexCandidates_GenerateNoteResults()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithVexCandidate();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-VEX-001" &&
|
||||
r.Level == SarifLevel.Note);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Reachability changes included when option enabled")]
|
||||
public void ReachabilityChanges_IncludedWhenEnabled()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithReachabilityChange();
|
||||
var options = new SarifOutputOptions { IncludeReachabilityChanges = true };
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input, options);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-REACH-001");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Reachability changes excluded when option disabled")]
|
||||
public void ReachabilityChanges_ExcludedWhenDisabled()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithReachabilityChange();
|
||||
var options = new SarifOutputOptions { IncludeReachabilityChanges = false };
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input, options);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().NotContain(r =>
|
||||
r.RuleId == "SDIFF-REACH-001");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Tool driver contains rule definitions")]
|
||||
public void ToolDriver_ContainsRuleDefinitions()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
var rules = sarifLog.Runs[0].Tool.Driver.Rules;
|
||||
rules.Should().NotBeNull();
|
||||
rules!.Value.Should().Contain(r => r.Id == "SDIFF-RISK-001");
|
||||
rules!.Value.Should().Contain(r => r.Id == "SDIFF-HARDENING-001");
|
||||
rules!.Value.Should().Contain(r => r.Id == "SDIFF-VEX-001");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "VCS provenance included when provided")]
|
||||
public void VcsProvenance_IncludedWhenProvided()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithVcs();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].VersionControlProvenance.Should().NotBeNull();
|
||||
sarifLog.Runs[0].VersionControlProvenance!.Value.Should().HaveCount(1);
|
||||
sarifLog.Runs[0].VersionControlProvenance!.Value[0].RepositoryUri
|
||||
.Should().Be("https://github.com/example/repo");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Invocation records scan time")]
|
||||
public void Invocation_RecordsScanTime()
|
||||
{
|
||||
// Arrange
|
||||
var scanTime = new DateTimeOffset(2025, 12, 17, 10, 0, 0, TimeSpan.Zero);
|
||||
var input = new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: scanTime,
|
||||
BaseDigest: "sha256:base",
|
||||
TargetDigest: "sha256:target",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Invocations.Should().NotBeNull();
|
||||
sarifLog.Runs[0].Invocations!.Value[0].StartTimeUtc.Should().Be("2025-12-17T10:00:00Z");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests (SDIFF-BIN-027)
|
||||
|
||||
[Fact(DisplayName = "Output is deterministic for same input")]
|
||||
public void Output_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarif1 = _generator.Generate(input);
|
||||
var sarif2 = _generator.Generate(input);
|
||||
|
||||
var json1 = JsonSerializer.Serialize(sarif1, JsonOptions);
|
||||
var json2 = JsonSerializer.Serialize(sarif2, JsonOptions);
|
||||
|
||||
// Assert
|
||||
json1.Should().Be(json2, "SARIF output should be deterministic for the same input");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Result order is stable")]
|
||||
public void ResultOrder_IsStable()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithMultipleFindings();
|
||||
|
||||
// Act - generate multiple times
|
||||
var results = Enumerable.Range(0, 5)
|
||||
.Select(_ => _generator.Generate(input).Runs[0].Results)
|
||||
.ToList();
|
||||
|
||||
// Assert - all result orders should match
|
||||
var firstOrder = results[0].Select(r => r.RuleId + r.Message.Text).ToList();
|
||||
foreach (var resultSet in results.Skip(1))
|
||||
{
|
||||
var order = resultSet.Select(r => r.RuleId + r.Message.Text).ToList();
|
||||
order.Should().Equal(firstOrder, "Result order should be stable across generations");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Golden fixture: basic SARIF output matches expected")]
|
||||
public void GoldenFixture_BasicSarif_MatchesExpected()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateGoldenFixtureInput();
|
||||
var expected = GetExpectedGoldenOutput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
var actual = JsonSerializer.Serialize(sarifLog, JsonOptions);
|
||||
|
||||
// Assert - normalize for comparison
|
||||
var actualNormalized = NormalizeJson(actual);
|
||||
var expectedNormalized = NormalizeJson(expected);
|
||||
|
||||
actualNormalized.Should().Be(expectedNormalized,
|
||||
"Generated SARIF should match golden fixture");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static JsonSchema GetSarifSchema()
|
||||
{
|
||||
// Inline minimal SARIF 2.1.0 schema for testing
|
||||
// In production, this would load the full schema from resources
|
||||
var schemaJson = """
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"required": ["version", "$schema", "runs"],
|
||||
"properties": {
|
||||
"version": { "const": "2.1.0" },
|
||||
"$schema": { "type": "string" },
|
||||
"runs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["tool", "results"],
|
||||
"properties": {
|
||||
"tool": {
|
||||
"type": "object",
|
||||
"required": ["driver"],
|
||||
"properties": {
|
||||
"driver": {
|
||||
"type": "object",
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"version": { "type": "string" },
|
||||
"informationUri": { "type": "string" },
|
||||
"rules": { "type": "array" }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["ruleId", "level", "message"],
|
||||
"properties": {
|
||||
"ruleId": { "type": "string" },
|
||||
"level": { "enum": ["none", "note", "warning", "error"] },
|
||||
"message": {
|
||||
"type": "object",
|
||||
"required": ["text"],
|
||||
"properties": {
|
||||
"text": { "type": "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
""";
|
||||
return JsonSchema.FromText(schemaJson);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateEmptyInput()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:base",
|
||||
TargetDigest: "sha256:target",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateBasicInput()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges:
|
||||
[
|
||||
new MaterialRiskChange(
|
||||
VulnId: "CVE-2025-0001",
|
||||
ComponentPurl: "pkg:npm/lodash@4.17.20",
|
||||
Direction: RiskDirection.Increased,
|
||||
Reason: "New vulnerability introduced")
|
||||
],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithHardeningRegression()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions:
|
||||
[
|
||||
new HardeningRegression(
|
||||
BinaryPath: "/usr/bin/app",
|
||||
FlagName: "PIE",
|
||||
WasEnabled: true,
|
||||
IsEnabled: false,
|
||||
ScoreImpact: -0.2)
|
||||
],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithVexCandidate()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates:
|
||||
[
|
||||
new VexCandidate(
|
||||
VulnId: "CVE-2025-0002",
|
||||
ComponentPurl: "pkg:npm/express@4.18.0",
|
||||
Justification: "not_affected",
|
||||
ImpactStatement: "Vulnerable code path not reachable")
|
||||
],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithReachabilityChange()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges:
|
||||
[
|
||||
new ReachabilityChange(
|
||||
VulnId: "CVE-2025-0003",
|
||||
ComponentPurl: "pkg:npm/axios@0.21.0",
|
||||
WasReachable: false,
|
||||
IsReachable: true,
|
||||
Evidence: "Call path: main -> http.get -> axios.request")
|
||||
]);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithVcs()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: [],
|
||||
VcsInfo: new VcsInfo(
|
||||
RepositoryUri: "https://github.com/example/repo",
|
||||
RevisionId: "abc123def456",
|
||||
Branch: "main"));
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithMultipleFindings()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: new DateTimeOffset(2025, 12, 17, 10, 0, 0, TimeSpan.Zero),
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges:
|
||||
[
|
||||
new MaterialRiskChange("CVE-2025-0001", "pkg:npm/a@1.0.0", RiskDirection.Increased, "Test 1"),
|
||||
new MaterialRiskChange("CVE-2025-0002", "pkg:npm/b@1.0.0", RiskDirection.Decreased, "Test 2"),
|
||||
new MaterialRiskChange("CVE-2025-0003", "pkg:npm/c@1.0.0", RiskDirection.Changed, "Test 3")
|
||||
],
|
||||
HardeningRegressions:
|
||||
[
|
||||
new HardeningRegression("/bin/app1", "PIE", true, false, -0.1),
|
||||
new HardeningRegression("/bin/app2", "RELRO", true, false, -0.1)
|
||||
],
|
||||
VexCandidates:
|
||||
[
|
||||
new VexCandidate("CVE-2025-0004", "pkg:npm/d@1.0.0", "not_affected", "Impact 1"),
|
||||
new VexCandidate("CVE-2025-0005", "pkg:npm/e@1.0.0", "vulnerable_code_not_in_execute_path", "Impact 2")
|
||||
],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateGoldenFixtureInput()
|
||||
{
|
||||
// Fixed input for golden fixture comparison
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0-golden",
|
||||
ScanTime: new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
||||
BaseDigest: "sha256:golden-base",
|
||||
TargetDigest: "sha256:golden-target",
|
||||
MaterialChanges:
|
||||
[
|
||||
new MaterialRiskChange("CVE-2025-GOLDEN", "pkg:npm/golden@1.0.0", RiskDirection.Increased, "Golden test finding")
|
||||
],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static string GetExpectedGoldenOutput()
|
||||
{
|
||||
// Expected golden output for determinism testing
|
||||
// This would typically be stored as a resource file
|
||||
return """
|
||||
{
|
||||
"version": "2.1.0",
|
||||
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "StellaOps.Scanner.SmartDiff",
|
||||
"version": "1.0.0-golden",
|
||||
"informationUri": "https://stellaops.dev/docs/scanner/smart-diff",
|
||||
"rules": []
|
||||
}
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"ruleId": "SDIFF-RISK-001",
|
||||
"level": "warning",
|
||||
"message": {
|
||||
"text": "Material risk change: CVE-2025-GOLDEN in pkg:npm/golden@1.0.0 - Golden test finding"
|
||||
}
|
||||
}
|
||||
],
|
||||
"invocations": [
|
||||
{
|
||||
"executionSuccessful": true,
|
||||
"startTimeUtc": "2025-01-01T00:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
}
|
||||
|
||||
private static string NormalizeJson(string json)
|
||||
{
|
||||
// Normalize JSON for comparison by parsing and re-serializing
|
||||
var doc = JsonDocument.Parse(json);
|
||||
return JsonSerializer.Serialize(doc.RootElement, new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user