feat(rate-limiting): Implement core rate limiting functionality with configuration, decision-making, metrics, middleware, and service registration
- Add RateLimitConfig for configuration management with YAML binding support. - Introduce RateLimitDecision to encapsulate the result of rate limit checks. - Implement RateLimitMetrics for OpenTelemetry metrics tracking. - Create RateLimitMiddleware for enforcing rate limits on incoming requests. - Develop RateLimitService to orchestrate instance and environment rate limit checks. - Add RateLimitServiceCollectionExtensions for dependency injection registration.
This commit is contained in:
@@ -0,0 +1,497 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ElfHardeningExtractorTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-022 - Unit tests for ELF hardening extraction
|
||||
// Description: Tests for ELF binary hardening flag detection
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Buffers.Binary;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Scanner.Analyzers.Native.Hardening;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Native.Tests.Hardening;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for ELF hardening flag extraction.
|
||||
/// Tests PIE, RELRO, NX, Stack Canary, and FORTIFY detection.
|
||||
/// </summary>
|
||||
public class ElfHardeningExtractorTests
|
||||
{
|
||||
private readonly ElfHardeningExtractor _extractor = new();
|
||||
|
||||
#region Magic Detection Tests
|
||||
|
||||
[Fact]
|
||||
public void CanExtract_ValidElfMagic_ReturnsTrue()
|
||||
{
|
||||
// Arrange - ELF magic: \x7FELF
|
||||
var header = new byte[] { 0x7F, 0x45, 0x4C, 0x46, 0x02, 0x01, 0x01, 0x00 };
|
||||
|
||||
// Act
|
||||
var result = _extractor.CanExtract(header);
|
||||
|
||||
// Assert
|
||||
result.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanExtract_InvalidMagic_ReturnsFalse()
|
||||
{
|
||||
// Arrange - Not ELF magic
|
||||
var header = new byte[] { 0x4D, 0x5A, 0x90, 0x00 }; // PE magic
|
||||
|
||||
// Act
|
||||
var result = _extractor.CanExtract(header);
|
||||
|
||||
// Assert
|
||||
result.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanExtract_TooShort_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
var header = new byte[] { 0x7F, 0x45 };
|
||||
|
||||
// Act
|
||||
var result = _extractor.CanExtract(header);
|
||||
|
||||
// Assert
|
||||
result.Should().BeFalse();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region PIE Detection Tests (SDIFF-BIN-004)
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_EtDynWithDtFlags1Pie_DetectsPie()
|
||||
{
|
||||
// Arrange - 64-bit ELF with ET_DYN type and DT_FLAGS_1 with DF_1_PIE
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3, // ET_DYN
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(2, 0, 1000, 200), // PT_DYNAMIC
|
||||
},
|
||||
dynamicEntries: new[]
|
||||
{
|
||||
(0x6ffffffbUL, 0x08000000UL), // DT_FLAGS_1 = DF_1_PIE
|
||||
(0UL, 0UL) // DT_NULL
|
||||
});
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var pieFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Pie);
|
||||
pieFlag.Should().NotBeNull();
|
||||
pieFlag!.Enabled.Should().BeTrue();
|
||||
pieFlag.Source.Should().Contain("DT_FLAGS_1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_EtExec_DoesNotDetectPie()
|
||||
{
|
||||
// Arrange - 64-bit ELF with ET_EXEC type (not PIE)
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 2, // ET_EXEC
|
||||
programHeaders: Array.Empty<byte[]>(),
|
||||
dynamicEntries: Array.Empty<(ulong, ulong)>());
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var pieFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Pie);
|
||||
pieFlag.Should().NotBeNull();
|
||||
pieFlag!.Enabled.Should().BeFalse();
|
||||
result.MissingFlags.Should().Contain("PIE");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region NX Detection Tests (SDIFF-BIN-006)
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_GnuStackNoExecute_DetectsNx()
|
||||
{
|
||||
// Arrange - PT_GNU_STACK without PF_X flag
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e551, 6, 0, 0), // PT_GNU_STACK with PF_R|PF_W (no PF_X)
|
||||
},
|
||||
dynamicEntries: new[] { (0UL, 0UL) });
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var nxFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Nx);
|
||||
nxFlag.Should().NotBeNull();
|
||||
nxFlag!.Enabled.Should().BeTrue();
|
||||
nxFlag.Source.Should().Contain("PT_GNU_STACK");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_GnuStackWithExecute_DoesNotDetectNx()
|
||||
{
|
||||
// Arrange - PT_GNU_STACK with PF_X flag (executable stack)
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e551, 7, 0, 0), // PT_GNU_STACK with PF_R|PF_W|PF_X
|
||||
},
|
||||
dynamicEntries: new[] { (0UL, 0UL) });
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var nxFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Nx);
|
||||
nxFlag.Should().NotBeNull();
|
||||
nxFlag!.Enabled.Should().BeFalse();
|
||||
result.MissingFlags.Should().Contain("NX");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_NoGnuStack_AssumesNx()
|
||||
{
|
||||
// Arrange - No PT_GNU_STACK (modern default is NX)
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: Array.Empty<byte[]>(),
|
||||
dynamicEntries: new[] { (0UL, 0UL) });
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var nxFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Nx);
|
||||
nxFlag.Should().NotBeNull();
|
||||
nxFlag!.Enabled.Should().BeTrue();
|
||||
nxFlag.Source.Should().Contain("assumed");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region RELRO Detection Tests (SDIFF-BIN-005)
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_GnuRelroOnly_DetectsPartialRelro()
|
||||
{
|
||||
// Arrange - PT_GNU_RELRO without BIND_NOW
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e552, 4, 0, 4096), // PT_GNU_RELRO
|
||||
CreateProgramHeader64(2, 0, 1000, 200), // PT_DYNAMIC
|
||||
},
|
||||
dynamicEntries: new[] { (0UL, 0UL) }); // No BIND_NOW
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var partialRelro = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.RelroPartial);
|
||||
partialRelro.Should().NotBeNull();
|
||||
partialRelro!.Enabled.Should().BeTrue();
|
||||
|
||||
var fullRelro = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.RelroFull);
|
||||
fullRelro.Should().NotBeNull();
|
||||
fullRelro!.Enabled.Should().BeFalse();
|
||||
result.MissingFlags.Should().Contain("RELRO_FULL");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_GnuRelroWithBindNow_DetectsFullRelro()
|
||||
{
|
||||
// Arrange - PT_GNU_RELRO with DT_FLAGS_1 containing DF_1_NOW
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e552, 4, 0, 4096), // PT_GNU_RELRO
|
||||
CreateProgramHeader64(2, 0, 1000, 200), // PT_DYNAMIC
|
||||
},
|
||||
dynamicEntries: new[]
|
||||
{
|
||||
(0x6ffffffbUL, 0x00000001UL), // DT_FLAGS_1 = DF_1_NOW
|
||||
(0UL, 0UL)
|
||||
});
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var partialRelro = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.RelroPartial);
|
||||
partialRelro!.Enabled.Should().BeTrue();
|
||||
|
||||
var fullRelro = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.RelroFull);
|
||||
fullRelro!.Enabled.Should().BeTrue();
|
||||
fullRelro.Source.Should().Contain("BIND_NOW");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_NoGnuRelro_DetectsNoRelro()
|
||||
{
|
||||
// Arrange - No PT_GNU_RELRO
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(2, 0, 1000, 200), // PT_DYNAMIC only
|
||||
},
|
||||
dynamicEntries: new[] { (0UL, 0UL) });
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var partialRelro = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.RelroPartial);
|
||||
partialRelro!.Enabled.Should().BeFalse();
|
||||
result.MissingFlags.Should().Contain("RELRO_PARTIAL");
|
||||
result.MissingFlags.Should().Contain("RELRO_FULL");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Hardening Score Tests (SDIFF-BIN-024)
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_AllHardeningEnabled_ReturnsHighScore()
|
||||
{
|
||||
// Arrange - PIE + NX enabled
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3, // ET_DYN (PIE)
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e551, 6, 0, 0), // PT_GNU_STACK (NX)
|
||||
CreateProgramHeader64(0x6474e552, 4, 0, 4096), // PT_GNU_RELRO
|
||||
CreateProgramHeader64(2, 0, 1000, 200), // PT_DYNAMIC
|
||||
},
|
||||
dynamicEntries: new[]
|
||||
{
|
||||
(0x6ffffffbUL, 0x08000001UL), // DT_FLAGS_1 = DF_1_PIE | DF_1_NOW
|
||||
(0UL, 0UL)
|
||||
});
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert - PIE, NX, RELRO_FULL enabled = 3/5 = 0.6
|
||||
result.HardeningScore.Should().BeGreaterOrEqualTo(0.6);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_NoHardening_ReturnsLowScore()
|
||||
{
|
||||
// Arrange - ET_EXEC, executable stack
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 2, // ET_EXEC (no PIE)
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e551, 7, 0, 0), // PT_GNU_STACK with PF_X
|
||||
},
|
||||
dynamicEntries: new[] { (0UL, 0UL) });
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
result.HardeningScore.Should().BeLessThan(0.5);
|
||||
result.MissingFlags.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region RPATH Detection Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_HasRpath_FlagsAsSecurityRisk()
|
||||
{
|
||||
// Arrange - DT_RPATH present
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(2, 0, 1000, 200), // PT_DYNAMIC
|
||||
},
|
||||
dynamicEntries: new[]
|
||||
{
|
||||
(15UL, 100UL), // DT_RPATH
|
||||
(0UL, 0UL)
|
||||
});
|
||||
|
||||
using var stream = new MemoryStream(elfData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var rpathFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Rpath);
|
||||
rpathFlag.Should().NotBeNull();
|
||||
rpathFlag!.Enabled.Should().BeTrue(); // true means RPATH is present (bad)
|
||||
rpathFlag.Value.Should().Contain("security risk");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_SameInput_ReturnsSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var elfData = CreateMinimalElf64(
|
||||
eType: 3,
|
||||
programHeaders: new[]
|
||||
{
|
||||
CreateProgramHeader64(0x6474e551, 6, 0, 0),
|
||||
CreateProgramHeader64(0x6474e552, 4, 0, 4096),
|
||||
CreateProgramHeader64(2, 0, 1000, 200),
|
||||
},
|
||||
dynamicEntries: new[]
|
||||
{
|
||||
(0x6ffffffbUL, 0x08000001UL),
|
||||
(0UL, 0UL)
|
||||
});
|
||||
|
||||
// Act - run extraction multiple times
|
||||
using var stream1 = new MemoryStream(elfData);
|
||||
var result1 = await _extractor.ExtractAsync(stream1, "/test/binary", "sha256:test");
|
||||
|
||||
using var stream2 = new MemoryStream(elfData);
|
||||
var result2 = await _extractor.ExtractAsync(stream2, "/test/binary", "sha256:test");
|
||||
|
||||
using var stream3 = new MemoryStream(elfData);
|
||||
var result3 = await _extractor.ExtractAsync(stream3, "/test/binary", "sha256:test");
|
||||
|
||||
// Assert - all results should have same flags (except timestamp)
|
||||
result1.HardeningScore.Should().Be(result2.HardeningScore);
|
||||
result2.HardeningScore.Should().Be(result3.HardeningScore);
|
||||
result1.Flags.Length.Should().Be(result2.Flags.Length);
|
||||
result2.Flags.Length.Should().Be(result3.Flags.Length);
|
||||
|
||||
for (int i = 0; i < result1.Flags.Length; i++)
|
||||
{
|
||||
result1.Flags[i].Name.Should().Be(result2.Flags[i].Name);
|
||||
result1.Flags[i].Enabled.Should().Be(result2.Flags[i].Enabled);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static byte[] CreateMinimalElf64(
|
||||
ushort eType,
|
||||
byte[][] programHeaders,
|
||||
(ulong tag, ulong value)[] dynamicEntries)
|
||||
{
|
||||
// Create a minimal valid 64-bit ELF structure
|
||||
var elfHeader = new byte[64];
|
||||
|
||||
// ELF magic
|
||||
elfHeader[0] = 0x7F;
|
||||
elfHeader[1] = 0x45; // E
|
||||
elfHeader[2] = 0x4C; // L
|
||||
elfHeader[3] = 0x46; // F
|
||||
|
||||
// EI_CLASS = ELFCLASS64
|
||||
elfHeader[4] = 2;
|
||||
// EI_DATA = ELFDATA2LSB (little endian)
|
||||
elfHeader[5] = 1;
|
||||
// EI_VERSION
|
||||
elfHeader[6] = 1;
|
||||
|
||||
// e_type (offset 16)
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(elfHeader.AsSpan(16), eType);
|
||||
|
||||
// e_machine (offset 18) - x86-64
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(elfHeader.AsSpan(18), 0x3E);
|
||||
|
||||
// e_phoff (offset 32) - program header offset
|
||||
var phOffset = 64UL;
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(elfHeader.AsSpan(32), phOffset);
|
||||
|
||||
// e_phentsize (offset 54) - 56 bytes for 64-bit
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(elfHeader.AsSpan(54), 56);
|
||||
|
||||
// e_phnum (offset 56)
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(elfHeader.AsSpan(56), (ushort)programHeaders.Length);
|
||||
|
||||
// Build the full ELF
|
||||
var result = new List<byte>(elfHeader);
|
||||
|
||||
// Add program headers
|
||||
foreach (var ph in programHeaders)
|
||||
{
|
||||
result.AddRange(ph);
|
||||
}
|
||||
|
||||
// Pad to offset 1000 for dynamic section
|
||||
while (result.Count < 1000)
|
||||
{
|
||||
result.Add(0);
|
||||
}
|
||||
|
||||
// Add dynamic entries
|
||||
foreach (var (tag, value) in dynamicEntries)
|
||||
{
|
||||
var entry = new byte[16];
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(entry.AsSpan(0, 8), tag);
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(entry.AsSpan(8, 8), value);
|
||||
result.AddRange(entry);
|
||||
}
|
||||
|
||||
return result.ToArray();
|
||||
}
|
||||
|
||||
private static byte[] CreateProgramHeader64(uint type, uint flags, ulong offset, ulong fileSize)
|
||||
{
|
||||
var ph = new byte[56];
|
||||
|
||||
// p_type (offset 0)
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(ph.AsSpan(0, 4), type);
|
||||
// p_flags (offset 4)
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(ph.AsSpan(4, 4), flags);
|
||||
// p_offset (offset 8)
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(ph.AsSpan(8, 8), offset);
|
||||
// p_vaddr (offset 16)
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(ph.AsSpan(16, 8), offset);
|
||||
// p_filesz (offset 32)
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(ph.AsSpan(32, 8), fileSize);
|
||||
// p_memsz (offset 40)
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(ph.AsSpan(40, 8), fileSize);
|
||||
|
||||
return ph;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// HardeningScoreCalculatorTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-024 - Unit tests for hardening score calculation
|
||||
// Description: Tests for hardening score calculation edge cases
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Scanner.Analyzers.Native.Hardening;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Native.Tests.Hardening;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for hardening score calculation.
|
||||
/// </summary>
|
||||
public class HardeningScoreCalculatorTests
|
||||
{
|
||||
#region Score Range Tests
|
||||
|
||||
[Fact]
|
||||
public void Score_AllFlagsEnabled_ReturnsOneOrNearOne()
|
||||
{
|
||||
// Arrange - all positive flags enabled
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, true),
|
||||
new HardeningFlag(HardeningFlagType.RelroFull, true),
|
||||
new HardeningFlag(HardeningFlagType.Nx, true),
|
||||
new HardeningFlag(HardeningFlagType.StackCanary, true),
|
||||
new HardeningFlag(HardeningFlagType.Fortify, true)
|
||||
);
|
||||
|
||||
var result = new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: "/test/binary",
|
||||
Digest: "sha256:test",
|
||||
Flags: flags,
|
||||
HardeningScore: CalculateScore(flags, BinaryFormat.Elf),
|
||||
MissingFlags: [],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
// Assert
|
||||
result.HardeningScore.Should().BeGreaterOrEqualTo(0.8);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_NoFlagsEnabled_ReturnsZero()
|
||||
{
|
||||
// Arrange - all flags disabled
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, false),
|
||||
new HardeningFlag(HardeningFlagType.RelroFull, false),
|
||||
new HardeningFlag(HardeningFlagType.Nx, false),
|
||||
new HardeningFlag(HardeningFlagType.StackCanary, false),
|
||||
new HardeningFlag(HardeningFlagType.Fortify, false)
|
||||
);
|
||||
|
||||
var result = new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: "/test/binary",
|
||||
Digest: "sha256:test",
|
||||
Flags: flags,
|
||||
HardeningScore: CalculateScore(flags, BinaryFormat.Elf),
|
||||
MissingFlags: ["PIE", "RELRO", "NX", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
// Assert
|
||||
result.HardeningScore.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_EmptyFlags_ReturnsZero()
|
||||
{
|
||||
// Arrange
|
||||
var flags = ImmutableArray<HardeningFlag>.Empty;
|
||||
|
||||
var result = new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: "/test/binary",
|
||||
Digest: "sha256:test",
|
||||
Flags: flags,
|
||||
HardeningScore: CalculateScore(flags, BinaryFormat.Elf),
|
||||
MissingFlags: [],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
// Assert
|
||||
result.HardeningScore.Should().Be(0);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1, 5, 0.2)]
|
||||
[InlineData(2, 5, 0.4)]
|
||||
[InlineData(3, 5, 0.6)]
|
||||
[InlineData(4, 5, 0.8)]
|
||||
[InlineData(5, 5, 1.0)]
|
||||
public void Score_PartialFlags_ReturnsProportionalScore(int enabled, int total, double expected)
|
||||
{
|
||||
// Arrange
|
||||
var flagTypes = new[]
|
||||
{
|
||||
HardeningFlagType.Pie,
|
||||
HardeningFlagType.RelroFull,
|
||||
HardeningFlagType.Nx,
|
||||
HardeningFlagType.StackCanary,
|
||||
HardeningFlagType.Fortify
|
||||
};
|
||||
|
||||
var flags = flagTypes.Take(total).Select((t, i) => new HardeningFlag(t, i < enabled)).ToImmutableArray();
|
||||
|
||||
var score = CalculateScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score.Should().BeApproximately(expected, 0.01);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Format-Specific Tests
|
||||
|
||||
[Fact]
|
||||
public void Score_ElfFormat_UsesElfPositiveFlags()
|
||||
{
|
||||
// Arrange - ELF-specific flags
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, true),
|
||||
new HardeningFlag(HardeningFlagType.RelroFull, true),
|
||||
new HardeningFlag(HardeningFlagType.Nx, true),
|
||||
new HardeningFlag(HardeningFlagType.StackCanary, true),
|
||||
new HardeningFlag(HardeningFlagType.Fortify, true),
|
||||
new HardeningFlag(HardeningFlagType.Rpath, false) // RPATH is negative - presence is bad
|
||||
);
|
||||
|
||||
var score = CalculateScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert - should be 1.0 (5/5 positive flags enabled)
|
||||
score.Should().Be(1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_PeFormat_UsesPePositiveFlags()
|
||||
{
|
||||
// Arrange - PE-specific flags
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Aslr, true),
|
||||
new HardeningFlag(HardeningFlagType.Dep, true),
|
||||
new HardeningFlag(HardeningFlagType.Cfg, true),
|
||||
new HardeningFlag(HardeningFlagType.Authenticode, true),
|
||||
new HardeningFlag(HardeningFlagType.Gs, true)
|
||||
);
|
||||
|
||||
var score = CalculateScore(flags, BinaryFormat.Pe);
|
||||
|
||||
// Assert
|
||||
score.Should().Be(1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_MachoFormat_UsesMachoPositiveFlags()
|
||||
{
|
||||
// Arrange - Mach-O specific flags
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, true),
|
||||
new HardeningFlag(HardeningFlagType.Nx, true),
|
||||
new HardeningFlag(HardeningFlagType.Authenticode, true), // Code signing
|
||||
new HardeningFlag(HardeningFlagType.Restrict, true)
|
||||
);
|
||||
|
||||
var score = CalculateScore(flags, BinaryFormat.MachO);
|
||||
|
||||
// Assert
|
||||
score.Should().Be(1.0);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Edge Cases
|
||||
|
||||
[Fact]
|
||||
public void Score_OnlyNegativeFlags_ReturnsZero()
|
||||
{
|
||||
// Arrange - only negative flags (RPATH is presence = bad)
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Rpath, true) // Enabled but not counted as positive
|
||||
);
|
||||
|
||||
var score = CalculateScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_MixedPositiveNegative_OnlyCountsPositive()
|
||||
{
|
||||
// Arrange
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, true),
|
||||
new HardeningFlag(HardeningFlagType.Nx, true),
|
||||
new HardeningFlag(HardeningFlagType.Rpath, true), // Negative flag
|
||||
new HardeningFlag(HardeningFlagType.RelroFull, false),
|
||||
new HardeningFlag(HardeningFlagType.StackCanary, false),
|
||||
new HardeningFlag(HardeningFlagType.Fortify, false)
|
||||
);
|
||||
|
||||
var score = CalculateScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert - 2 positive enabled out of 5
|
||||
score.Should().BeApproximately(0.4, 0.01);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_RelroPartial_CountsLessThanFull()
|
||||
{
|
||||
// RELRO partial should count as 0.5, full as 1.0
|
||||
var partialFlags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.RelroPartial, true),
|
||||
new HardeningFlag(HardeningFlagType.RelroFull, false)
|
||||
);
|
||||
|
||||
var fullFlags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.RelroPartial, false),
|
||||
new HardeningFlag(HardeningFlagType.RelroFull, true)
|
||||
);
|
||||
|
||||
var partialScore = CalculateScoreWithRelro(partialFlags);
|
||||
var fullScore = CalculateScoreWithRelro(fullFlags);
|
||||
|
||||
// Full RELRO should be better than partial
|
||||
fullScore.Should().BeGreaterThan(partialScore);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests
|
||||
|
||||
[Fact]
|
||||
public void Score_SameFlags_ReturnsSameScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, true),
|
||||
new HardeningFlag(HardeningFlagType.Nx, true)
|
||||
);
|
||||
|
||||
// Act - calculate multiple times
|
||||
var score1 = CalculateScore(flags, BinaryFormat.Elf);
|
||||
var score2 = CalculateScore(flags, BinaryFormat.Elf);
|
||||
var score3 = CalculateScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score1.Should().Be(score2);
|
||||
score2.Should().Be(score3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Score_DifferentFlagOrder_ReturnsSameScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags1 = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Pie, true),
|
||||
new HardeningFlag(HardeningFlagType.Nx, true)
|
||||
);
|
||||
|
||||
var flags2 = ImmutableArray.Create(
|
||||
new HardeningFlag(HardeningFlagType.Nx, true),
|
||||
new HardeningFlag(HardeningFlagType.Pie, true)
|
||||
);
|
||||
|
||||
// Act
|
||||
var score1 = CalculateScore(flags1, BinaryFormat.Elf);
|
||||
var score2 = CalculateScore(flags2, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score1.Should().Be(score2);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
/// <summary>
|
||||
/// Calculate score using the same logic as the extractors.
|
||||
/// </summary>
|
||||
private static double CalculateScore(ImmutableArray<HardeningFlag> flags, BinaryFormat format)
|
||||
{
|
||||
var positiveFlags = format switch
|
||||
{
|
||||
BinaryFormat.Elf => new[]
|
||||
{
|
||||
HardeningFlagType.Pie,
|
||||
HardeningFlagType.RelroFull,
|
||||
HardeningFlagType.Nx,
|
||||
HardeningFlagType.StackCanary,
|
||||
HardeningFlagType.Fortify
|
||||
},
|
||||
BinaryFormat.Pe => new[]
|
||||
{
|
||||
HardeningFlagType.Aslr,
|
||||
HardeningFlagType.Dep,
|
||||
HardeningFlagType.Cfg,
|
||||
HardeningFlagType.Authenticode,
|
||||
HardeningFlagType.Gs
|
||||
},
|
||||
BinaryFormat.MachO => new[]
|
||||
{
|
||||
HardeningFlagType.Pie,
|
||||
HardeningFlagType.Nx,
|
||||
HardeningFlagType.Authenticode,
|
||||
HardeningFlagType.Restrict
|
||||
},
|
||||
_ => Array.Empty<HardeningFlagType>()
|
||||
};
|
||||
|
||||
if (positiveFlags.Length == 0)
|
||||
return 0;
|
||||
|
||||
var enabledCount = flags.Count(f => f.Enabled && positiveFlags.Contains(f.Name));
|
||||
return Math.Round((double)enabledCount / positiveFlags.Length, 2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculate score with RELRO weighting.
|
||||
/// </summary>
|
||||
private static double CalculateScoreWithRelro(ImmutableArray<HardeningFlag> flags)
|
||||
{
|
||||
var score = 0.0;
|
||||
var total = 1.0; // Just RELRO for this test
|
||||
|
||||
var hasPartial = flags.Any(f => f.Name == HardeningFlagType.RelroPartial && f.Enabled);
|
||||
var hasFull = flags.Any(f => f.Name == HardeningFlagType.RelroFull && f.Enabled);
|
||||
|
||||
if (hasFull)
|
||||
score = 1.0;
|
||||
else if (hasPartial)
|
||||
score = 0.5;
|
||||
|
||||
return Math.Round(score / total, 2);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// HardeningScoringTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-024 - Unit tests for hardening score calculation
|
||||
// Description: Tests for hardening score calculation edge cases and determinism
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Scanner.Analyzers.Native.Hardening;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Native.Tests.Hardening;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for hardening score calculation.
|
||||
/// Tests score computation, edge cases, and determinism.
|
||||
/// </summary>
|
||||
public class HardeningScoringTests
|
||||
{
|
||||
#region Score Calculation Tests
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_AllFlagsEnabled_Returns1()
|
||||
{
|
||||
// Arrange - All critical flags enabled
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.RelroFull, true),
|
||||
(HardeningFlagType.Nx, true),
|
||||
(HardeningFlagType.StackCanary, true),
|
||||
(HardeningFlagType.Fortify, true));
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score.Should().BeApproximately(1.0, 0.01);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_NoFlagsEnabled_Returns0()
|
||||
{
|
||||
// Arrange - No flags enabled
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Pie, false),
|
||||
(HardeningFlagType.RelroFull, false),
|
||||
(HardeningFlagType.Nx, false),
|
||||
(HardeningFlagType.StackCanary, false),
|
||||
(HardeningFlagType.Fortify, false));
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score.Should().Be(0.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_PartialFlags_ReturnsProportionalScore()
|
||||
{
|
||||
// Arrange - Only PIE and NX enabled (2 of 5 critical flags)
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Nx, true),
|
||||
(HardeningFlagType.RelroFull, false),
|
||||
(HardeningFlagType.StackCanary, false),
|
||||
(HardeningFlagType.Fortify, false));
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score.Should().BeGreaterThan(0.0);
|
||||
score.Should().BeLessThan(1.0);
|
||||
// With equal weights: 2/5 = 0.4
|
||||
score.Should().BeApproximately(0.4, 0.1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Edge Case Tests
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_EmptyFlags_Returns0()
|
||||
{
|
||||
// Arrange
|
||||
var flags = ImmutableArray<HardeningFlag>.Empty;
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score.Should().Be(0.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_UnknownFormat_ReturnsBasedOnAvailableFlags()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Nx, true));
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.Unknown);
|
||||
|
||||
// Assert
|
||||
score.Should().BeGreaterThan(0.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_PartialRelro_CountsLessThanFullRelro()
|
||||
{
|
||||
// Arrange
|
||||
var flagsPartial = CreateFlags(
|
||||
(HardeningFlagType.RelroPartial, true),
|
||||
(HardeningFlagType.RelroFull, false));
|
||||
|
||||
var flagsFull = CreateFlags(
|
||||
(HardeningFlagType.RelroPartial, true),
|
||||
(HardeningFlagType.RelroFull, true));
|
||||
|
||||
// Act
|
||||
var scorePartial = CalculateHardeningScore(flagsPartial, BinaryFormat.Elf);
|
||||
var scoreFull = CalculateHardeningScore(flagsFull, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
scoreFull.Should().BeGreaterThan(scorePartial);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_RpathPresent_ReducesScore()
|
||||
{
|
||||
// Arrange - RPATH is a negative indicator
|
||||
var flagsNoRpath = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Rpath, false));
|
||||
|
||||
var flagsWithRpath = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Rpath, true));
|
||||
|
||||
// Act
|
||||
var scoreNoRpath = CalculateHardeningScore(flagsNoRpath, BinaryFormat.Elf);
|
||||
var scoreWithRpath = CalculateHardeningScore(flagsWithRpath, BinaryFormat.Elf);
|
||||
|
||||
// Assert - RPATH presence should reduce or not improve score
|
||||
scoreWithRpath.Should().BeLessThanOrEqualTo(scoreNoRpath);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_SameInput_AlwaysReturnsSameScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Nx, true),
|
||||
(HardeningFlagType.StackCanary, true));
|
||||
|
||||
// Act - Calculate multiple times
|
||||
var scores = Enumerable.Range(0, 100)
|
||||
.Select(_ => CalculateHardeningScore(flags, BinaryFormat.Elf))
|
||||
.ToList();
|
||||
|
||||
// Assert - All scores should be identical
|
||||
scores.Should().AllBeEquivalentTo(scores[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_FlagOrderDoesNotMatter()
|
||||
{
|
||||
// Arrange - Same flags in different orders
|
||||
var flags1 = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Nx, true),
|
||||
(HardeningFlagType.StackCanary, true));
|
||||
|
||||
var flags2 = CreateFlags(
|
||||
(HardeningFlagType.StackCanary, true),
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Nx, true));
|
||||
|
||||
var flags3 = CreateFlags(
|
||||
(HardeningFlagType.Nx, true),
|
||||
(HardeningFlagType.StackCanary, true),
|
||||
(HardeningFlagType.Pie, true));
|
||||
|
||||
// Act
|
||||
var score1 = CalculateHardeningScore(flags1, BinaryFormat.Elf);
|
||||
var score2 = CalculateHardeningScore(flags2, BinaryFormat.Elf);
|
||||
var score3 = CalculateHardeningScore(flags3, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
score1.Should().Be(score2);
|
||||
score2.Should().Be(score3);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Format-Specific Tests
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_PeFormat_UsesCorrectFlags()
|
||||
{
|
||||
// Arrange - PE-specific flags
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Aslr, true),
|
||||
(HardeningFlagType.Dep, true),
|
||||
(HardeningFlagType.Cfg, true),
|
||||
(HardeningFlagType.Authenticode, true),
|
||||
(HardeningFlagType.SafeSeh, true));
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.Pe);
|
||||
|
||||
// Assert
|
||||
score.Should().BeApproximately(1.0, 0.01);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_MachOFormat_UsesCorrectFlags()
|
||||
{
|
||||
// Arrange - Mach-O specific flags
|
||||
var flags = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Hardened, true),
|
||||
(HardeningFlagType.CodeSign, true),
|
||||
(HardeningFlagType.LibraryValidation, true));
|
||||
|
||||
// Act
|
||||
var score = CalculateHardeningScore(flags, BinaryFormat.MachO);
|
||||
|
||||
// Assert
|
||||
score.Should().BeApproximately(1.0, 0.01);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CET/BTI Tests (Task SDIFF-BIN-009)
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_CetEnabled_IncreasesScore()
|
||||
{
|
||||
// Arrange
|
||||
var flagsWithoutCet = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Cet, false));
|
||||
|
||||
var flagsWithCet = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Cet, true));
|
||||
|
||||
// Act
|
||||
var scoreWithoutCet = CalculateHardeningScore(flagsWithoutCet, BinaryFormat.Elf);
|
||||
var scoreWithCet = CalculateHardeningScore(flagsWithCet, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
scoreWithCet.Should().BeGreaterThan(scoreWithoutCet);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HardeningScore_BtiEnabled_IncreasesScore()
|
||||
{
|
||||
// Arrange
|
||||
var flagsWithoutBti = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Bti, false));
|
||||
|
||||
var flagsWithBti = CreateFlags(
|
||||
(HardeningFlagType.Pie, true),
|
||||
(HardeningFlagType.Bti, true));
|
||||
|
||||
// Act
|
||||
var scoreWithoutBti = CalculateHardeningScore(flagsWithoutBti, BinaryFormat.Elf);
|
||||
var scoreWithBti = CalculateHardeningScore(flagsWithBti, BinaryFormat.Elf);
|
||||
|
||||
// Assert
|
||||
scoreWithBti.Should().BeGreaterThan(scoreWithoutBti);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helpers
|
||||
|
||||
private static ImmutableArray<HardeningFlag> CreateFlags(params (HardeningFlagType Type, bool Enabled)[] flags)
|
||||
{
|
||||
return flags.Select(f => new HardeningFlag(f.Type, f.Enabled)).ToImmutableArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculate hardening score based on enabled flags.
|
||||
/// This mirrors the production scoring logic.
|
||||
/// </summary>
|
||||
private static double CalculateHardeningScore(ImmutableArray<HardeningFlag> flags, BinaryFormat format)
|
||||
{
|
||||
if (flags.IsEmpty)
|
||||
return 0.0;
|
||||
|
||||
// Define weights for each flag type
|
||||
var weights = GetWeightsForFormat(format);
|
||||
|
||||
double totalWeight = 0;
|
||||
double enabledWeight = 0;
|
||||
|
||||
foreach (var flag in flags)
|
||||
{
|
||||
if (weights.TryGetValue(flag.Name, out var weight))
|
||||
{
|
||||
// RPATH is a negative indicator - invert the logic
|
||||
if (flag.Name == HardeningFlagType.Rpath)
|
||||
{
|
||||
totalWeight += weight;
|
||||
if (!flag.Enabled) // RPATH absent is good
|
||||
enabledWeight += weight;
|
||||
}
|
||||
else
|
||||
{
|
||||
totalWeight += weight;
|
||||
if (flag.Enabled)
|
||||
enabledWeight += weight;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return totalWeight > 0 ? enabledWeight / totalWeight : 0.0;
|
||||
}
|
||||
|
||||
private static Dictionary<HardeningFlagType, double> GetWeightsForFormat(BinaryFormat format)
|
||||
{
|
||||
return format switch
|
||||
{
|
||||
BinaryFormat.Elf => new Dictionary<HardeningFlagType, double>
|
||||
{
|
||||
[HardeningFlagType.Pie] = 1.0,
|
||||
[HardeningFlagType.RelroPartial] = 0.5,
|
||||
[HardeningFlagType.RelroFull] = 1.0,
|
||||
[HardeningFlagType.Nx] = 1.0,
|
||||
[HardeningFlagType.StackCanary] = 1.0,
|
||||
[HardeningFlagType.Fortify] = 1.0,
|
||||
[HardeningFlagType.Rpath] = 0.5,
|
||||
[HardeningFlagType.Cet] = 0.75,
|
||||
[HardeningFlagType.Bti] = 0.75
|
||||
},
|
||||
BinaryFormat.Pe => new Dictionary<HardeningFlagType, double>
|
||||
{
|
||||
[HardeningFlagType.Aslr] = 1.0,
|
||||
[HardeningFlagType.Dep] = 1.0,
|
||||
[HardeningFlagType.Cfg] = 1.0,
|
||||
[HardeningFlagType.Authenticode] = 1.0,
|
||||
[HardeningFlagType.SafeSeh] = 1.0,
|
||||
[HardeningFlagType.Gs] = 0.75,
|
||||
[HardeningFlagType.HighEntropyVa] = 0.5,
|
||||
[HardeningFlagType.ForceIntegrity] = 0.5
|
||||
},
|
||||
BinaryFormat.MachO => new Dictionary<HardeningFlagType, double>
|
||||
{
|
||||
[HardeningFlagType.Pie] = 1.0,
|
||||
[HardeningFlagType.Hardened] = 1.0,
|
||||
[HardeningFlagType.CodeSign] = 1.0,
|
||||
[HardeningFlagType.LibraryValidation] = 1.0,
|
||||
[HardeningFlagType.Restrict] = 0.5
|
||||
},
|
||||
_ => new Dictionary<HardeningFlagType, double>
|
||||
{
|
||||
[HardeningFlagType.Pie] = 1.0,
|
||||
[HardeningFlagType.Nx] = 1.0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// PeHardeningExtractorTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-023 - Unit tests for PE hardening extraction
|
||||
// Description: Tests for PE binary hardening flag detection
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Buffers.Binary;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Scanner.Analyzers.Native.Hardening;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Native.Tests.Hardening;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for PE hardening flag extraction.
|
||||
/// Tests ASLR, DEP, CFG, Authenticode, and other security features.
|
||||
/// </summary>
|
||||
public class PeHardeningExtractorTests
|
||||
{
|
||||
private readonly PeHardeningExtractor _extractor = new();
|
||||
|
||||
#region Magic Detection Tests
|
||||
|
||||
[Fact]
|
||||
public void CanExtract_ValidPeMagic_ReturnsTrue()
|
||||
{
|
||||
// Arrange - PE magic: MZ
|
||||
var header = new byte[] { 0x4D, 0x5A, 0x90, 0x00 };
|
||||
|
||||
// Act
|
||||
var result = _extractor.CanExtract(header);
|
||||
|
||||
// Assert
|
||||
result.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanExtract_InvalidMagic_ReturnsFalse()
|
||||
{
|
||||
// Arrange - Not PE magic (ELF)
|
||||
var header = new byte[] { 0x7F, 0x45, 0x4C, 0x46 };
|
||||
|
||||
// Act
|
||||
var result = _extractor.CanExtract(header);
|
||||
|
||||
// Assert
|
||||
result.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanExtract_TooShort_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
var header = new byte[] { 0x4D };
|
||||
|
||||
// Act
|
||||
var result = _extractor.CanExtract(header);
|
||||
|
||||
// Assert
|
||||
result.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(".exe", true)]
|
||||
[InlineData(".dll", true)]
|
||||
[InlineData(".sys", true)]
|
||||
[InlineData(".ocx", true)]
|
||||
[InlineData(".EXE", true)]
|
||||
[InlineData(".txt", false)]
|
||||
[InlineData(".so", false)]
|
||||
public void CanExtract_ByPath_ChecksExtension(string extension, bool expected)
|
||||
{
|
||||
// Act
|
||||
var result = _extractor.CanExtract($"test{extension}");
|
||||
|
||||
// Assert
|
||||
result.Should().Be(expected);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region DllCharacteristics Flag Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_AslrEnabled_DetectsAslr()
|
||||
{
|
||||
// Arrange - PE32+ with DYNAMIC_BASE flag
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x0040); // DYNAMIC_BASE
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var aslrFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Aslr);
|
||||
aslrFlag.Should().NotBeNull();
|
||||
aslrFlag!.Enabled.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_DepEnabled_DetectsDep()
|
||||
{
|
||||
// Arrange - PE32+ with NX_COMPAT flag
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x0100); // NX_COMPAT
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var depFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Dep);
|
||||
depFlag.Should().NotBeNull();
|
||||
depFlag!.Enabled.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_CfgEnabled_DetectsCfg()
|
||||
{
|
||||
// Arrange - PE32+ with GUARD_CF flag
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x4000); // GUARD_CF
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var cfgFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Cfg);
|
||||
cfgFlag.Should().NotBeNull();
|
||||
cfgFlag!.Enabled.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_HighEntropyVa_DetectsHighEntropyVa()
|
||||
{
|
||||
// Arrange - PE32+ with HIGH_ENTROPY_VA flag
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x0020); // HIGH_ENTROPY_VA
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var hevaFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.HighEntropyVa);
|
||||
hevaFlag.Should().NotBeNull();
|
||||
hevaFlag!.Enabled.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_AllFlagsEnabled_HighScore()
|
||||
{
|
||||
// Arrange - PE32+ with all hardening flags
|
||||
ushort allFlags = 0x0040 | 0x0020 | 0x0100 | 0x4000; // ASLR + HIGH_ENTROPY + DEP + CFG
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: allFlags, hasSecurityDir: true, hasLoadConfig: true);
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
result.HardeningScore.Should().BeGreaterOrEqualTo(0.8);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_NoFlags_LowScore()
|
||||
{
|
||||
// Arrange - PE32+ with no hardening flags
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x0000);
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
result.HardeningScore.Should().BeLessThan(0.5);
|
||||
result.MissingFlags.Should().Contain("ASLR");
|
||||
result.MissingFlags.Should().Contain("DEP");
|
||||
result.MissingFlags.Should().Contain("CFG");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Authenticode Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_WithAuthenticode_DetectsSigning()
|
||||
{
|
||||
// Arrange - PE with security directory
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x0040, hasSecurityDir: true);
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var authFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Authenticode);
|
||||
authFlag.Should().NotBeNull();
|
||||
authFlag!.Enabled.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_NoAuthenticode_FlagsAsMissing()
|
||||
{
|
||||
// Arrange - PE without security directory
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x0040, hasSecurityDir: false);
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
var authFlag = result.Flags.FirstOrDefault(f => f.Name == HardeningFlagType.Authenticode);
|
||||
authFlag.Should().NotBeNull();
|
||||
authFlag!.Enabled.Should().BeFalse();
|
||||
result.MissingFlags.Should().Contain("AUTHENTICODE");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Invalid PE Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_TooSmall_ReturnsError()
|
||||
{
|
||||
// Arrange - Too small to be a valid PE
|
||||
var peData = new byte[32];
|
||||
peData[0] = 0x4D;
|
||||
peData[1] = 0x5A;
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
result.Flags.Should().BeEmpty();
|
||||
result.MissingFlags.Should().Contain(s => s.Contains("Invalid"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_BadDosMagic_ReturnsError()
|
||||
{
|
||||
// Arrange - Wrong DOS magic
|
||||
var peData = new byte[512];
|
||||
peData[0] = 0x00;
|
||||
peData[1] = 0x00;
|
||||
|
||||
using var stream = new MemoryStream(peData);
|
||||
|
||||
// Act
|
||||
var result = await _extractor.ExtractAsync(stream, "test.exe", "sha256:test");
|
||||
|
||||
// Assert
|
||||
result.MissingFlags.Should().Contain(s => s.Contains("DOS magic"));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractAsync_SameInput_ReturnsSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var peData = CreateMinimalPe64(dllCharacteristics: 0x4140, hasSecurityDir: true);
|
||||
|
||||
// Act - run extraction multiple times
|
||||
using var stream1 = new MemoryStream(peData);
|
||||
var result1 = await _extractor.ExtractAsync(stream1, "test.exe", "sha256:test");
|
||||
|
||||
using var stream2 = new MemoryStream(peData);
|
||||
var result2 = await _extractor.ExtractAsync(stream2, "test.exe", "sha256:test");
|
||||
|
||||
using var stream3 = new MemoryStream(peData);
|
||||
var result3 = await _extractor.ExtractAsync(stream3, "test.exe", "sha256:test");
|
||||
|
||||
// Assert - all results should have same flags
|
||||
result1.HardeningScore.Should().Be(result2.HardeningScore);
|
||||
result2.HardeningScore.Should().Be(result3.HardeningScore);
|
||||
result1.Flags.Length.Should().Be(result2.Flags.Length);
|
||||
result2.Flags.Length.Should().Be(result3.Flags.Length);
|
||||
|
||||
for (int i = 0; i < result1.Flags.Length; i++)
|
||||
{
|
||||
result1.Flags[i].Name.Should().Be(result2.Flags[i].Name);
|
||||
result1.Flags[i].Enabled.Should().Be(result2.Flags[i].Enabled);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
/// <summary>
|
||||
/// Create a minimal valid PE64 (PE32+) structure for testing.
|
||||
/// </summary>
|
||||
private static byte[] CreateMinimalPe64(
|
||||
ushort dllCharacteristics,
|
||||
bool hasSecurityDir = false,
|
||||
bool hasLoadConfig = false)
|
||||
{
|
||||
// Create a minimal PE file structure
|
||||
var pe = new byte[512];
|
||||
|
||||
// DOS Header
|
||||
pe[0] = 0x4D; // M
|
||||
pe[1] = 0x5A; // Z
|
||||
BinaryPrimitives.WriteInt32LittleEndian(pe.AsSpan(0x3C), 0x80); // e_lfanew = PE header at 0x80
|
||||
|
||||
// PE Signature at offset 0x80
|
||||
pe[0x80] = 0x50; // P
|
||||
pe[0x81] = 0x45; // E
|
||||
pe[0x82] = 0x00;
|
||||
pe[0x83] = 0x00;
|
||||
|
||||
// COFF Header at 0x84
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(pe.AsSpan(0x84), 0x8664); // AMD64 machine
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(pe.AsSpan(0x86), 1); // 1 section
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(pe.AsSpan(0x94), 240); // Size of optional header
|
||||
|
||||
// Optional Header at 0x98
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(pe.AsSpan(0x98), 0x20B); // PE32+ magic
|
||||
|
||||
// DllCharacteristics at offset 0x98 + 70 = 0xDE
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(pe.AsSpan(0xDE), dllCharacteristics);
|
||||
|
||||
// NumberOfRvaAndSizes at 0x98 + 108 = 0x104
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(pe.AsSpan(0x104), 16);
|
||||
|
||||
// Data Directories start at 0x98 + 112 = 0x108
|
||||
// Security Directory (index 4) at 0x108 + 32 = 0x128
|
||||
if (hasSecurityDir)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(pe.AsSpan(0x128), 0x1000); // RVA
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(pe.AsSpan(0x12C), 256); // Size
|
||||
}
|
||||
|
||||
// Load Config Directory (index 10) at 0x108 + 80 = 0x158
|
||||
if (hasLoadConfig)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(pe.AsSpan(0x158), 0x2000); // RVA
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(pe.AsSpan(0x15C), 256); // Size
|
||||
}
|
||||
|
||||
return pe;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
// =============================================================================
|
||||
// CorpusRunnerIntegrationTests.cs
|
||||
// Sprint: SPRINT_3500_0003_0001_ground_truth_corpus_ci_gates
|
||||
// Task: CORPUS-013 - Integration tests for corpus runner
|
||||
// =============================================================================
|
||||
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Moq;
|
||||
using StellaOps.Scanner.Benchmarks;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Benchmarks.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for the ground-truth corpus runner.
|
||||
/// Per Sprint 3500.0003.0001 - Ground-Truth Corpus & CI Regression Gates.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Sprint", "3500.3")]
|
||||
public sealed class CorpusRunnerIntegrationTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
#region Corpus Runner Tests
|
||||
|
||||
[Fact(DisplayName = "RunAsync produces valid benchmark result")]
|
||||
public async Task RunAsync_ProducesValidBenchmarkResult()
|
||||
{
|
||||
// Arrange
|
||||
var runner = new MockCorpusRunner();
|
||||
var corpusPath = "TestData/corpus.json";
|
||||
var options = new CorpusRunOptions();
|
||||
|
||||
// Act
|
||||
var result = await runner.RunAsync(corpusPath, options);
|
||||
|
||||
// Assert
|
||||
result.Should().NotBeNull();
|
||||
result.RunId.Should().NotBeNullOrEmpty();
|
||||
result.Timestamp.Should().BeCloseTo(DateTimeOffset.UtcNow, TimeSpan.FromMinutes(1));
|
||||
result.CorpusVersion.Should().NotBeNullOrEmpty();
|
||||
result.ScannerVersion.Should().NotBeNullOrEmpty();
|
||||
result.Metrics.Should().NotBeNull();
|
||||
result.SampleResults.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "RunAsync computes correct metrics")]
|
||||
public async Task RunAsync_ComputesCorrectMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var runner = new MockCorpusRunner(
|
||||
truePositives: 8,
|
||||
falsePositives: 1,
|
||||
falseNegatives: 1);
|
||||
var options = new CorpusRunOptions();
|
||||
|
||||
// Act
|
||||
var result = await runner.RunAsync("TestData/corpus.json", options);
|
||||
|
||||
// Assert - 8 TP, 1 FP, 1 FN = precision 8/9 = 0.8889, recall 8/9 = 0.8889
|
||||
result.Metrics.Precision.Should().BeApproximately(0.8889, 0.01);
|
||||
result.Metrics.Recall.Should().BeApproximately(0.8889, 0.01);
|
||||
result.Metrics.F1.Should().BeApproximately(0.8889, 0.01);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "RunAsync respects category filter")]
|
||||
public async Task RunAsync_RespectsFilter()
|
||||
{
|
||||
// Arrange
|
||||
var runner = new MockCorpusRunner(sampleCount: 20);
|
||||
var options = new CorpusRunOptions { Categories = ["basic"] };
|
||||
|
||||
// Act
|
||||
var result = await runner.RunAsync("TestData/corpus.json", options);
|
||||
|
||||
// Assert
|
||||
result.SampleResults.Should().OnlyContain(r => r.Category == "basic");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "RunAsync handles timeout correctly")]
|
||||
public async Task RunAsync_HandlesTimeout()
|
||||
{
|
||||
// Arrange
|
||||
var runner = new MockCorpusRunner(sampleLatencyMs: 5000);
|
||||
var options = new CorpusRunOptions { TimeoutMs = 100 };
|
||||
|
||||
// Act
|
||||
var result = await runner.RunAsync("TestData/corpus.json", options);
|
||||
|
||||
// Assert
|
||||
result.SampleResults.Should().OnlyContain(r => r.Error != null);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "RunAsync performs determinism checks")]
|
||||
public async Task RunAsync_PerformsDeterminismChecks()
|
||||
{
|
||||
// Arrange
|
||||
var runner = new MockCorpusRunner(deterministicRate: 1.0);
|
||||
var options = new CorpusRunOptions
|
||||
{
|
||||
CheckDeterminism = true,
|
||||
DeterminismRuns = 3
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await runner.RunAsync("TestData/corpus.json", options);
|
||||
|
||||
// Assert
|
||||
result.Metrics.DeterministicReplay.Should().Be(1.0);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Metrics Computation Tests
|
||||
|
||||
[Fact(DisplayName = "BenchmarkMetrics.Compute calculates precision correctly")]
|
||||
public void BenchmarkMetrics_Compute_CalculatesPrecisionCorrectly()
|
||||
{
|
||||
// Arrange - 7 TP, 3 FP => precision = 7/10 = 0.7
|
||||
var sinkResults = new List<SinkResult>
|
||||
{
|
||||
// True positives
|
||||
new("s1", "reachable", "reachable", true),
|
||||
new("s2", "reachable", "reachable", true),
|
||||
new("s3", "reachable", "reachable", true),
|
||||
new("s4", "reachable", "reachable", true),
|
||||
new("s5", "reachable", "reachable", true),
|
||||
new("s6", "reachable", "reachable", true),
|
||||
new("s7", "reachable", "reachable", true),
|
||||
// False positives
|
||||
new("s8", "unreachable", "reachable", false),
|
||||
new("s9", "unreachable", "reachable", false),
|
||||
new("s10", "unreachable", "reachable", false),
|
||||
};
|
||||
|
||||
var sample = new SampleResult("test-001", "Test", "basic", sinkResults, 100, true);
|
||||
var results = new List<SampleResult> { sample };
|
||||
|
||||
// Act
|
||||
var metrics = BenchmarkMetrics.Compute(results);
|
||||
|
||||
// Assert
|
||||
metrics.Precision.Should().BeApproximately(0.7, 0.01);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "BenchmarkMetrics.Compute calculates recall correctly")]
|
||||
public void BenchmarkMetrics_Compute_CalculatesRecallCorrectly()
|
||||
{
|
||||
// Arrange - 8 TP, 2 FN => recall = 8/10 = 0.8
|
||||
var sinkResults = new List<SinkResult>
|
||||
{
|
||||
// True positives
|
||||
new("s1", "reachable", "reachable", true),
|
||||
new("s2", "reachable", "reachable", true),
|
||||
new("s3", "reachable", "reachable", true),
|
||||
new("s4", "reachable", "reachable", true),
|
||||
new("s5", "reachable", "reachable", true),
|
||||
new("s6", "reachable", "reachable", true),
|
||||
new("s7", "reachable", "reachable", true),
|
||||
new("s8", "reachable", "reachable", true),
|
||||
// False negatives
|
||||
new("s9", "reachable", "unreachable", false),
|
||||
new("s10", "reachable", "unreachable", false),
|
||||
};
|
||||
|
||||
var sample = new SampleResult("test-001", "Test", "basic", sinkResults, 100, true);
|
||||
var results = new List<SampleResult> { sample };
|
||||
|
||||
// Act
|
||||
var metrics = BenchmarkMetrics.Compute(results);
|
||||
|
||||
// Assert
|
||||
metrics.Recall.Should().BeApproximately(0.8, 0.01);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "BenchmarkMetrics.Compute calculates F1 correctly")]
|
||||
public void BenchmarkMetrics_Compute_CalculatesF1Correctly()
|
||||
{
|
||||
// Arrange - precision 0.8, recall 0.6 => F1 = 2*0.8*0.6/(0.8+0.6) ≈ 0.686
|
||||
var sinkResults = new List<SinkResult>
|
||||
{
|
||||
// 8 TP, 2 FP => precision = 0.8
|
||||
// 8 TP, 5.33 FN => recall = 0.6 (adjusting for F1)
|
||||
// Let's use: 6 TP, 4 FN => recall = 0.6; 6 TP, 1.5 FP => precision = 0.8
|
||||
// Actually: 4 TP, 1 FP (precision = 0.8), 4 TP, 2.67 FN (not integer)
|
||||
// Simpler: 8 TP, 2 FP, 2 FN => P=0.8, R=0.8, F1=0.8
|
||||
new("s1", "reachable", "reachable", true),
|
||||
new("s2", "reachable", "reachable", true),
|
||||
new("s3", "reachable", "reachable", true),
|
||||
new("s4", "reachable", "reachable", true),
|
||||
new("s5", "reachable", "reachable", true),
|
||||
new("s6", "reachable", "reachable", true),
|
||||
new("s7", "reachable", "reachable", true),
|
||||
new("s8", "reachable", "reachable", true),
|
||||
new("s9", "unreachable", "reachable", false), // FP
|
||||
new("s10", "unreachable", "reachable", false), // FP
|
||||
new("s11", "reachable", "unreachable", false), // FN
|
||||
new("s12", "reachable", "unreachable", false), // FN
|
||||
};
|
||||
|
||||
var sample = new SampleResult("test-001", "Test", "basic", sinkResults, 100, true);
|
||||
var results = new List<SampleResult> { sample };
|
||||
|
||||
// Act
|
||||
var metrics = BenchmarkMetrics.Compute(results);
|
||||
|
||||
// Assert - P = 8/10 = 0.8, R = 8/10 = 0.8, F1 = 0.8
|
||||
metrics.F1.Should().BeApproximately(0.8, 0.01);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "BenchmarkMetrics.Compute handles empty results")]
|
||||
public void BenchmarkMetrics_Compute_HandlesEmptyResults()
|
||||
{
|
||||
// Arrange
|
||||
var results = new List<SampleResult>();
|
||||
|
||||
// Act
|
||||
var metrics = BenchmarkMetrics.Compute(results);
|
||||
|
||||
// Assert
|
||||
metrics.Precision.Should().Be(0);
|
||||
metrics.Recall.Should().Be(0);
|
||||
metrics.F1.Should().Be(0);
|
||||
metrics.DeterministicReplay.Should().Be(1.0);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Regression Check Tests
|
||||
|
||||
[Fact(DisplayName = "CheckRegression passes when metrics are above baseline")]
|
||||
public void CheckRegression_PassesWhenAboveBaseline()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = new BenchmarkBaseline(
|
||||
Version: "1.0.0",
|
||||
Timestamp: DateTimeOffset.UtcNow.AddDays(-7),
|
||||
Precision: 0.90,
|
||||
Recall: 0.85,
|
||||
F1: 0.875,
|
||||
TtfrpP95Ms: 400);
|
||||
|
||||
var result = CreateBenchmarkResult(
|
||||
precision: 0.92,
|
||||
recall: 0.87,
|
||||
deterministicReplay: 1.0,
|
||||
ttfrpP95Ms: 350);
|
||||
|
||||
// Act
|
||||
var check = result.CheckRegression(baseline);
|
||||
|
||||
// Assert
|
||||
check.Passed.Should().BeTrue();
|
||||
check.Issues.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "CheckRegression fails on precision drop > 1pp")]
|
||||
public void CheckRegression_FailsOnPrecisionDrop()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = new BenchmarkBaseline(
|
||||
Version: "1.0.0",
|
||||
Timestamp: DateTimeOffset.UtcNow.AddDays(-7),
|
||||
Precision: 0.95,
|
||||
Recall: 0.90,
|
||||
F1: 0.924,
|
||||
TtfrpP95Ms: 400);
|
||||
|
||||
var result = CreateBenchmarkResult(
|
||||
precision: 0.92, // 3pp drop
|
||||
recall: 0.90,
|
||||
deterministicReplay: 1.0,
|
||||
ttfrpP95Ms: 400);
|
||||
|
||||
// Act
|
||||
var check = result.CheckRegression(baseline);
|
||||
|
||||
// Assert
|
||||
check.Passed.Should().BeFalse();
|
||||
check.Issues.Should().Contain(i => i.Metric == "precision" && i.Severity == RegressionSeverity.Error);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "CheckRegression fails on recall drop > 1pp")]
|
||||
public void CheckRegression_FailsOnRecallDrop()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = new BenchmarkBaseline(
|
||||
Version: "1.0.0",
|
||||
Timestamp: DateTimeOffset.UtcNow.AddDays(-7),
|
||||
Precision: 0.90,
|
||||
Recall: 0.95,
|
||||
F1: 0.924,
|
||||
TtfrpP95Ms: 400);
|
||||
|
||||
var result = CreateBenchmarkResult(
|
||||
precision: 0.90,
|
||||
recall: 0.92, // 3pp drop
|
||||
deterministicReplay: 1.0,
|
||||
ttfrpP95Ms: 400);
|
||||
|
||||
// Act
|
||||
var check = result.CheckRegression(baseline);
|
||||
|
||||
// Assert
|
||||
check.Passed.Should().BeFalse();
|
||||
check.Issues.Should().Contain(i => i.Metric == "recall" && i.Severity == RegressionSeverity.Error);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "CheckRegression fails on non-deterministic replay")]
|
||||
public void CheckRegression_FailsOnNonDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = new BenchmarkBaseline(
|
||||
Version: "1.0.0",
|
||||
Timestamp: DateTimeOffset.UtcNow.AddDays(-7),
|
||||
Precision: 0.90,
|
||||
Recall: 0.90,
|
||||
F1: 0.90,
|
||||
TtfrpP95Ms: 400);
|
||||
|
||||
var result = CreateBenchmarkResult(
|
||||
precision: 0.90,
|
||||
recall: 0.90,
|
||||
deterministicReplay: 0.95, // Not 100%
|
||||
ttfrpP95Ms: 400);
|
||||
|
||||
// Act
|
||||
var check = result.CheckRegression(baseline);
|
||||
|
||||
// Assert
|
||||
check.Passed.Should().BeFalse();
|
||||
check.Issues.Should().Contain(i => i.Metric == "determinism" && i.Severity == RegressionSeverity.Error);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "CheckRegression warns on TTFRP increase > 20%")]
|
||||
public void CheckRegression_WarnsOnTtfrpIncrease()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = new BenchmarkBaseline(
|
||||
Version: "1.0.0",
|
||||
Timestamp: DateTimeOffset.UtcNow.AddDays(-7),
|
||||
Precision: 0.90,
|
||||
Recall: 0.90,
|
||||
F1: 0.90,
|
||||
TtfrpP95Ms: 400);
|
||||
|
||||
var result = CreateBenchmarkResult(
|
||||
precision: 0.90,
|
||||
recall: 0.90,
|
||||
deterministicReplay: 1.0,
|
||||
ttfrpP95Ms: 520); // 30% increase
|
||||
|
||||
// Act
|
||||
var check = result.CheckRegression(baseline);
|
||||
|
||||
// Assert
|
||||
check.Passed.Should().BeTrue(); // Warning doesn't fail
|
||||
check.Issues.Should().Contain(i => i.Metric == "ttfrp_p95" && i.Severity == RegressionSeverity.Warning);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Serialization Tests
|
||||
|
||||
[Fact(DisplayName = "BenchmarkResult serializes to valid JSON")]
|
||||
public void BenchmarkResult_SerializesToValidJson()
|
||||
{
|
||||
// Arrange
|
||||
var result = CreateBenchmarkResult();
|
||||
|
||||
// Act
|
||||
var json = JsonSerializer.Serialize(result, JsonOptions);
|
||||
var deserialized = JsonSerializer.Deserialize<BenchmarkResult>(json, JsonOptions);
|
||||
|
||||
// Assert
|
||||
deserialized.Should().NotBeNull();
|
||||
deserialized!.RunId.Should().Be(result.RunId);
|
||||
deserialized.Metrics.Precision.Should().Be(result.Metrics.Precision);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "SampleResult serializes with correct property names")]
|
||||
public void SampleResult_SerializesWithCorrectPropertyNames()
|
||||
{
|
||||
// Arrange
|
||||
var sample = new SampleResult(
|
||||
"gt-0001",
|
||||
"test-sample",
|
||||
"basic",
|
||||
new[] { new SinkResult("sink-001", "reachable", "reachable", true) },
|
||||
150,
|
||||
true);
|
||||
|
||||
// Act
|
||||
var json = JsonSerializer.Serialize(sample, JsonOptions);
|
||||
|
||||
// Assert
|
||||
json.Should().Contain("\"sampleId\"");
|
||||
json.Should().Contain("\"latencyMs\"");
|
||||
json.Should().Contain("\"deterministic\"");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static BenchmarkResult CreateBenchmarkResult(
|
||||
double precision = 0.95,
|
||||
double recall = 0.92,
|
||||
double deterministicReplay = 1.0,
|
||||
int ttfrpP95Ms = 380)
|
||||
{
|
||||
var metrics = new BenchmarkMetrics(
|
||||
Precision: precision,
|
||||
Recall: recall,
|
||||
F1: 2 * precision * recall / (precision + recall),
|
||||
TtfrpP50Ms: 120,
|
||||
TtfrpP95Ms: ttfrpP95Ms,
|
||||
DeterministicReplay: deterministicReplay);
|
||||
|
||||
var sampleResults = new List<SampleResult>
|
||||
{
|
||||
new SampleResult("gt-0001", "sample-1", "basic",
|
||||
new[] { new SinkResult("sink-001", "reachable", "reachable", true) },
|
||||
120, true)
|
||||
};
|
||||
|
||||
return new BenchmarkResult(
|
||||
RunId: $"bench-{DateTimeOffset.UtcNow:yyyyMMdd}-001",
|
||||
Timestamp: DateTimeOffset.UtcNow,
|
||||
CorpusVersion: "1.0.0",
|
||||
ScannerVersion: "1.3.0",
|
||||
Metrics: metrics,
|
||||
SampleResults: sampleResults,
|
||||
DurationMs: 5000);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Mock Corpus Runner
|
||||
|
||||
private sealed class MockCorpusRunner : ICorpusRunner
|
||||
{
|
||||
private readonly int _truePositives;
|
||||
private readonly int _falsePositives;
|
||||
private readonly int _falseNegatives;
|
||||
private readonly int _sampleCount;
|
||||
private readonly int _sampleLatencyMs;
|
||||
private readonly double _deterministicRate;
|
||||
|
||||
public MockCorpusRunner(
|
||||
int truePositives = 9,
|
||||
int falsePositives = 0,
|
||||
int falseNegatives = 1,
|
||||
int sampleCount = 10,
|
||||
int sampleLatencyMs = 100,
|
||||
double deterministicRate = 1.0)
|
||||
{
|
||||
_truePositives = truePositives;
|
||||
_falsePositives = falsePositives;
|
||||
_falseNegatives = falseNegatives;
|
||||
_sampleCount = sampleCount;
|
||||
_sampleLatencyMs = sampleLatencyMs;
|
||||
_deterministicRate = deterministicRate;
|
||||
}
|
||||
|
||||
public Task<BenchmarkResult> RunAsync(string corpusPath, CorpusRunOptions options, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var samples = new List<SampleResult>();
|
||||
var random = new Random(42); // Deterministic seed
|
||||
|
||||
for (int i = 0; i < _sampleCount; i++)
|
||||
{
|
||||
var category = options.Categories?.FirstOrDefault() ?? "basic";
|
||||
var sinkResults = new List<SinkResult>();
|
||||
|
||||
if (i < _truePositives)
|
||||
{
|
||||
sinkResults.Add(new SinkResult($"sink-{i}", "reachable", "reachable", true));
|
||||
}
|
||||
else if (i < _truePositives + _falsePositives)
|
||||
{
|
||||
sinkResults.Add(new SinkResult($"sink-{i}", "unreachable", "reachable", false));
|
||||
}
|
||||
else if (i < _truePositives + _falsePositives + _falseNegatives)
|
||||
{
|
||||
sinkResults.Add(new SinkResult($"sink-{i}", "reachable", "unreachable", false));
|
||||
}
|
||||
else
|
||||
{
|
||||
sinkResults.Add(new SinkResult($"sink-{i}", "unreachable", "unreachable", true));
|
||||
}
|
||||
|
||||
var isDeterministic = random.NextDouble() < _deterministicRate;
|
||||
var error = _sampleLatencyMs > options.TimeoutMs ? "Timeout" : null;
|
||||
|
||||
samples.Add(new SampleResult(
|
||||
$"gt-{i:D4}",
|
||||
$"sample-{i}",
|
||||
category,
|
||||
sinkResults,
|
||||
_sampleLatencyMs,
|
||||
isDeterministic,
|
||||
error));
|
||||
}
|
||||
|
||||
var metrics = BenchmarkMetrics.Compute(samples);
|
||||
|
||||
var result = new BenchmarkResult(
|
||||
RunId: $"bench-{DateTimeOffset.UtcNow:yyyyMMddHHmmss}",
|
||||
Timestamp: DateTimeOffset.UtcNow,
|
||||
CorpusVersion: "1.0.0",
|
||||
ScannerVersion: "1.3.0-test",
|
||||
Metrics: metrics,
|
||||
SampleResults: samples,
|
||||
DurationMs: _sampleLatencyMs * samples.Count);
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<SampleResult> RunSampleAsync(string samplePath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new SampleResult(
|
||||
"gt-0001",
|
||||
"test-sample",
|
||||
"basic",
|
||||
new[] { new SinkResult("sink-001", "reachable", "reachable", true) },
|
||||
_sampleLatencyMs,
|
||||
true);
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="8.*" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.*" />
|
||||
<PackageReference Include="Moq" Version="4.*" />
|
||||
<PackageReference Include="xunit" Version="2.*" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.*">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Scanner.Benchmarks\StellaOps.Scanner.Benchmarks.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="TestData\**\*" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,269 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3500_0003_0001
|
||||
// Task: CORPUS-013 - Integration tests for corpus runner
|
||||
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Scanner.Reachability.Benchmarks;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability.Tests.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for the corpus runner and benchmark framework.
|
||||
/// </summary>
|
||||
public sealed class CorpusRunnerIntegrationTests
|
||||
{
|
||||
private static readonly string CorpusBasePath = Path.Combine(
|
||||
AppDomain.CurrentDomain.BaseDirectory,
|
||||
"..", "..", "..", "..", "..", "..", "..",
|
||||
"datasets", "reachability");
|
||||
|
||||
[Fact]
|
||||
public void CorpusIndex_ShouldBeValidJson()
|
||||
{
|
||||
// Arrange
|
||||
var corpusPath = Path.Combine(CorpusBasePath, "corpus.json");
|
||||
|
||||
if (!File.Exists(corpusPath))
|
||||
{
|
||||
// Skip if running outside of full repo context
|
||||
return;
|
||||
}
|
||||
|
||||
// Act
|
||||
var json = File.ReadAllText(corpusPath);
|
||||
var parseAction = () => JsonDocument.Parse(json);
|
||||
|
||||
// Assert
|
||||
parseAction.Should().NotThrow("corpus.json should be valid JSON");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CorpusIndex_ShouldContainRequiredFields()
|
||||
{
|
||||
// Arrange
|
||||
var corpusPath = Path.Combine(CorpusBasePath, "corpus.json");
|
||||
|
||||
if (!File.Exists(corpusPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Act
|
||||
var json = File.ReadAllText(corpusPath);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
|
||||
// Assert
|
||||
root.TryGetProperty("version", out _).Should().BeTrue("corpus should have version");
|
||||
root.TryGetProperty("samples", out var samples).Should().BeTrue("corpus should have samples");
|
||||
samples.GetArrayLength().Should().BeGreaterThan(0, "corpus should have at least one sample");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SampleManifest_ShouldHaveExpectedResult()
|
||||
{
|
||||
// Arrange
|
||||
var samplePath = Path.Combine(
|
||||
CorpusBasePath,
|
||||
"ground-truth", "basic", "gt-0001",
|
||||
"sample.manifest.json");
|
||||
|
||||
if (!File.Exists(samplePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Act
|
||||
var json = File.ReadAllText(samplePath);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
|
||||
// Assert
|
||||
root.TryGetProperty("sampleId", out var sampleId).Should().BeTrue();
|
||||
sampleId.GetString().Should().Be("gt-0001");
|
||||
|
||||
root.TryGetProperty("expectedResult", out var expectedResult).Should().BeTrue();
|
||||
expectedResult.TryGetProperty("reachable", out var reachable).Should().BeTrue();
|
||||
reachable.GetBoolean().Should().BeTrue("gt-0001 should be marked as reachable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnreachableSample_ShouldHaveFalseExpectedResult()
|
||||
{
|
||||
// Arrange
|
||||
var samplePath = Path.Combine(
|
||||
CorpusBasePath,
|
||||
"ground-truth", "unreachable", "gt-0011",
|
||||
"sample.manifest.json");
|
||||
|
||||
if (!File.Exists(samplePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Act
|
||||
var json = File.ReadAllText(samplePath);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
|
||||
// Assert
|
||||
root.TryGetProperty("sampleId", out var sampleId).Should().BeTrue();
|
||||
sampleId.GetString().Should().Be("gt-0011");
|
||||
|
||||
root.TryGetProperty("expectedResult", out var expectedResult).Should().BeTrue();
|
||||
expectedResult.TryGetProperty("reachable", out var reachable).Should().BeTrue();
|
||||
reachable.GetBoolean().Should().BeFalse("gt-0011 should be marked as unreachable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BenchmarkResult_ShouldCalculateMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var results = new List<SampleResult>
|
||||
{
|
||||
new("gt-0001", expected: true, actual: true, tier: "executed", durationMs: 10),
|
||||
new("gt-0002", expected: true, actual: true, tier: "executed", durationMs: 15),
|
||||
new("gt-0011", expected: false, actual: false, tier: "imported", durationMs: 5),
|
||||
new("gt-0012", expected: false, actual: true, tier: "executed", durationMs: 8), // False positive
|
||||
};
|
||||
|
||||
// Act
|
||||
var metrics = BenchmarkMetrics.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.TotalSamples.Should().Be(4);
|
||||
metrics.TruePositives.Should().Be(2);
|
||||
metrics.TrueNegatives.Should().Be(1);
|
||||
metrics.FalsePositives.Should().Be(1);
|
||||
metrics.FalseNegatives.Should().Be(0);
|
||||
metrics.Precision.Should().BeApproximately(0.666, 0.01);
|
||||
metrics.Recall.Should().Be(1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BenchmarkResult_ShouldDetectRegression()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = new BenchmarkMetrics
|
||||
{
|
||||
Precision = 0.95,
|
||||
Recall = 0.90,
|
||||
F1Score = 0.924,
|
||||
MeanDurationMs = 50
|
||||
};
|
||||
|
||||
var current = new BenchmarkMetrics
|
||||
{
|
||||
Precision = 0.85, // Dropped by 10%
|
||||
Recall = 0.92,
|
||||
F1Score = 0.883,
|
||||
MeanDurationMs = 55
|
||||
};
|
||||
|
||||
// Act
|
||||
var regressions = RegressionDetector.Check(baseline, current, thresholds: new()
|
||||
{
|
||||
MaxPrecisionDrop = 0.05,
|
||||
MaxRecallDrop = 0.05,
|
||||
MaxDurationIncrease = 0.20
|
||||
});
|
||||
|
||||
// Assert
|
||||
regressions.Should().Contain(r => r.Metric == "Precision");
|
||||
regressions.Should().NotContain(r => r.Metric == "Recall");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a single sample result from the benchmark run.
|
||||
/// </summary>
|
||||
public record SampleResult(
|
||||
string SampleId,
|
||||
bool Expected,
|
||||
bool Actual,
|
||||
string Tier,
|
||||
double DurationMs);
|
||||
|
||||
/// <summary>
|
||||
/// Calculated metrics from a benchmark run.
|
||||
/// </summary>
|
||||
public class BenchmarkMetrics
|
||||
{
|
||||
public int TotalSamples { get; set; }
|
||||
public int TruePositives { get; set; }
|
||||
public int TrueNegatives { get; set; }
|
||||
public int FalsePositives { get; set; }
|
||||
public int FalseNegatives { get; set; }
|
||||
public double Precision { get; set; }
|
||||
public double Recall { get; set; }
|
||||
public double F1Score { get; set; }
|
||||
public double MeanDurationMs { get; set; }
|
||||
|
||||
public static BenchmarkMetrics Calculate(IList<SampleResult> results)
|
||||
{
|
||||
var tp = results.Count(r => r.Expected && r.Actual);
|
||||
var tn = results.Count(r => !r.Expected && !r.Actual);
|
||||
var fp = results.Count(r => !r.Expected && r.Actual);
|
||||
var fn = results.Count(r => r.Expected && !r.Actual);
|
||||
|
||||
var precision = tp + fp > 0 ? (double)tp / (tp + fp) : 0;
|
||||
var recall = tp + fn > 0 ? (double)tp / (tp + fn) : 0;
|
||||
var f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0;
|
||||
|
||||
return new BenchmarkMetrics
|
||||
{
|
||||
TotalSamples = results.Count,
|
||||
TruePositives = tp,
|
||||
TrueNegatives = tn,
|
||||
FalsePositives = fp,
|
||||
FalseNegatives = fn,
|
||||
Precision = precision,
|
||||
Recall = recall,
|
||||
F1Score = f1,
|
||||
MeanDurationMs = results.Average(r => r.DurationMs)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Regression detector for benchmark comparisons.
|
||||
/// </summary>
|
||||
public static class RegressionDetector
|
||||
{
|
||||
public static List<Regression> Check(BenchmarkMetrics baseline, BenchmarkMetrics current, RegressionThresholds thresholds)
|
||||
{
|
||||
var regressions = new List<Regression>();
|
||||
|
||||
var precisionDrop = baseline.Precision - current.Precision;
|
||||
if (precisionDrop > thresholds.MaxPrecisionDrop)
|
||||
{
|
||||
regressions.Add(new Regression("Precision", baseline.Precision, current.Precision, precisionDrop));
|
||||
}
|
||||
|
||||
var recallDrop = baseline.Recall - current.Recall;
|
||||
if (recallDrop > thresholds.MaxRecallDrop)
|
||||
{
|
||||
regressions.Add(new Regression("Recall", baseline.Recall, current.Recall, recallDrop));
|
||||
}
|
||||
|
||||
var durationIncrease = (current.MeanDurationMs - baseline.MeanDurationMs) / baseline.MeanDurationMs;
|
||||
if (durationIncrease > thresholds.MaxDurationIncrease)
|
||||
{
|
||||
regressions.Add(new Regression("Duration", baseline.MeanDurationMs, current.MeanDurationMs, durationIncrease));
|
||||
}
|
||||
|
||||
return regressions;
|
||||
}
|
||||
}
|
||||
|
||||
public record Regression(string Metric, double Baseline, double Current, double Delta);
|
||||
|
||||
public class RegressionThresholds
|
||||
{
|
||||
public double MaxPrecisionDrop { get; set; } = 0.05;
|
||||
public double MaxRecallDrop { get; set; } = 0.05;
|
||||
public double MaxDurationIncrease { get; set; } = 0.20;
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3500_0001_0001
|
||||
// Task: SDIFF-MASTER-0007 - Performance benchmark suite
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Columns;
|
||||
using BenchmarkDotNet.Configs;
|
||||
using BenchmarkDotNet.Exporters;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using BenchmarkDotNet.Loggers;
|
||||
using BenchmarkDotNet.Running;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// BenchmarkDotNet performance benchmarks for Smart-Diff operations.
|
||||
/// Run with: dotnet run -c Release --project StellaOps.Scanner.SmartDiff.Tests.csproj -- --filter *SmartDiff*
|
||||
/// </summary>
|
||||
[Config(typeof(SmartDiffBenchmarkConfig))]
|
||||
[MemoryDiagnoser]
|
||||
[RankColumn]
|
||||
public class SmartDiffPerformanceBenchmarks
|
||||
{
|
||||
private ScanData _smallBaseline = null!;
|
||||
private ScanData _smallCurrent = null!;
|
||||
private ScanData _mediumBaseline = null!;
|
||||
private ScanData _mediumCurrent = null!;
|
||||
private ScanData _largeBaseline = null!;
|
||||
private ScanData _largeCurrent = null!;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
// Small: 50 packages, 10 vulnerabilities
|
||||
_smallBaseline = GenerateScanData(packageCount: 50, vulnCount: 10);
|
||||
_smallCurrent = GenerateScanData(packageCount: 55, vulnCount: 12, deltaPercent: 0.2);
|
||||
|
||||
// Medium: 500 packages, 100 vulnerabilities
|
||||
_mediumBaseline = GenerateScanData(packageCount: 500, vulnCount: 100);
|
||||
_mediumCurrent = GenerateScanData(packageCount: 520, vulnCount: 110, deltaPercent: 0.15);
|
||||
|
||||
// Large: 5000 packages, 1000 vulnerabilities
|
||||
_largeBaseline = GenerateScanData(packageCount: 5000, vulnCount: 1000);
|
||||
_largeCurrent = GenerateScanData(packageCount: 5100, vulnCount: 1050, deltaPercent: 0.10);
|
||||
}
|
||||
|
||||
[Benchmark(Baseline = true)]
|
||||
public DiffResult SmallScan_ComputeDiff()
|
||||
{
|
||||
return ComputeDiff(_smallBaseline, _smallCurrent);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public DiffResult MediumScan_ComputeDiff()
|
||||
{
|
||||
return ComputeDiff(_mediumBaseline, _mediumCurrent);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public DiffResult LargeScan_ComputeDiff()
|
||||
{
|
||||
return ComputeDiff(_largeBaseline, _largeCurrent);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public string SmallScan_GenerateSarif()
|
||||
{
|
||||
var diff = ComputeDiff(_smallBaseline, _smallCurrent);
|
||||
return GenerateSarif(diff);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public string MediumScan_GenerateSarif()
|
||||
{
|
||||
var diff = ComputeDiff(_mediumBaseline, _mediumCurrent);
|
||||
return GenerateSarif(diff);
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
public string LargeScan_GenerateSarif()
|
||||
{
|
||||
var diff = ComputeDiff(_largeBaseline, _largeCurrent);
|
||||
return GenerateSarif(diff);
|
||||
}
|
||||
|
||||
#region Benchmark Helpers
|
||||
|
||||
private static ScanData GenerateScanData(int packageCount, int vulnCount, double deltaPercent = 0)
|
||||
{
|
||||
var random = new Random(42); // Fixed seed for reproducibility
|
||||
var packages = new List<PackageInfo>();
|
||||
var vulnerabilities = new List<VulnInfo>();
|
||||
|
||||
for (int i = 0; i < packageCount; i++)
|
||||
{
|
||||
packages.Add(new PackageInfo
|
||||
{
|
||||
Name = $"package-{i:D5}",
|
||||
Version = $"1.{random.Next(0, 10)}.{random.Next(0, 100)}",
|
||||
Ecosystem = random.Next(0, 3) switch { 0 => "npm", 1 => "nuget", _ => "pypi" }
|
||||
});
|
||||
}
|
||||
|
||||
for (int i = 0; i < vulnCount; i++)
|
||||
{
|
||||
var pkg = packages[random.Next(0, packages.Count)];
|
||||
vulnerabilities.Add(new VulnInfo
|
||||
{
|
||||
CveId = $"CVE-2024-{10000 + i}",
|
||||
Package = pkg.Name,
|
||||
Version = pkg.Version,
|
||||
Severity = random.Next(0, 4) switch { 0 => "LOW", 1 => "MEDIUM", 2 => "HIGH", _ => "CRITICAL" },
|
||||
IsReachable = random.NextDouble() > 0.6,
|
||||
ReachabilityTier = random.Next(0, 3) switch { 0 => "imported", 1 => "called", _ => "executed" }
|
||||
});
|
||||
}
|
||||
|
||||
// Apply delta for current scans
|
||||
if (deltaPercent > 0)
|
||||
{
|
||||
int vulnsToAdd = (int)(vulnCount * deltaPercent);
|
||||
for (int i = 0; i < vulnsToAdd; i++)
|
||||
{
|
||||
var pkg = packages[random.Next(0, packages.Count)];
|
||||
vulnerabilities.Add(new VulnInfo
|
||||
{
|
||||
CveId = $"CVE-2024-{20000 + i}",
|
||||
Package = pkg.Name,
|
||||
Version = pkg.Version,
|
||||
Severity = "HIGH",
|
||||
IsReachable = true,
|
||||
ReachabilityTier = "executed"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new ScanData { Packages = packages, Vulnerabilities = vulnerabilities };
|
||||
}
|
||||
|
||||
private static DiffResult ComputeDiff(ScanData baseline, ScanData current)
|
||||
{
|
||||
var baselineSet = baseline.Vulnerabilities.ToHashSet(new VulnComparer());
|
||||
var currentSet = current.Vulnerabilities.ToHashSet(new VulnComparer());
|
||||
|
||||
var added = current.Vulnerabilities.Where(v => !baselineSet.Contains(v)).ToList();
|
||||
var removed = baseline.Vulnerabilities.Where(v => !currentSet.Contains(v)).ToList();
|
||||
|
||||
// Detect reachability flips
|
||||
var baselineDict = baseline.Vulnerabilities.ToDictionary(v => v.CveId);
|
||||
var reachabilityFlips = new List<VulnInfo>();
|
||||
foreach (var curr in current.Vulnerabilities)
|
||||
{
|
||||
if (baselineDict.TryGetValue(curr.CveId, out var prev) && prev.IsReachable != curr.IsReachable)
|
||||
{
|
||||
reachabilityFlips.Add(curr);
|
||||
}
|
||||
}
|
||||
|
||||
return new DiffResult
|
||||
{
|
||||
Added = added,
|
||||
Removed = removed,
|
||||
ReachabilityFlips = reachabilityFlips,
|
||||
TotalBaselineVulns = baseline.Vulnerabilities.Count,
|
||||
TotalCurrentVulns = current.Vulnerabilities.Count
|
||||
};
|
||||
}
|
||||
|
||||
private static string GenerateSarif(DiffResult diff)
|
||||
{
|
||||
var sarif = new
|
||||
{
|
||||
version = "2.1.0",
|
||||
schema = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
|
||||
runs = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
tool = new
|
||||
{
|
||||
driver = new
|
||||
{
|
||||
name = "StellaOps Smart-Diff",
|
||||
version = "1.0.0",
|
||||
informationUri = "https://stellaops.io"
|
||||
}
|
||||
},
|
||||
results = diff.Added.Select(v => new
|
||||
{
|
||||
ruleId = v.CveId,
|
||||
level = v.Severity == "CRITICAL" || v.Severity == "HIGH" ? "error" : "warning",
|
||||
message = new { text = $"New vulnerability {v.CveId} in {v.Package}@{v.Version}" },
|
||||
locations = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
physicalLocation = new
|
||||
{
|
||||
artifactLocation = new { uri = $"pkg:{v.Package}@{v.Version}" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}).ToArray()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(sarif, new JsonSerializerOptions { WriteIndented = false });
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performance threshold tests that fail CI if benchmarks regress.
|
||||
/// </summary>
|
||||
public sealed class SmartDiffPerformanceTests
|
||||
{
|
||||
[Fact]
|
||||
public void SmallScan_ShouldCompleteWithin50ms()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(50, 10);
|
||||
var current = GenerateTestData(55, 12);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = ComputeDiff(baseline, current);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(50, "Small scan diff should complete within 50ms");
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MediumScan_ShouldCompleteWithin200ms()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(500, 100);
|
||||
var current = GenerateTestData(520, 110);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = ComputeDiff(baseline, current);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(200, "Medium scan diff should complete within 200ms");
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LargeScan_ShouldCompleteWithin2000ms()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(5000, 1000);
|
||||
var current = GenerateTestData(5100, 1050);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = ComputeDiff(baseline, current);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(2000, "Large scan diff should complete within 2 seconds");
|
||||
result.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SarifGeneration_ShouldCompleteWithin100ms_ForSmallDiff()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(50, 10);
|
||||
var current = GenerateTestData(55, 15);
|
||||
var diff = ComputeDiff(baseline, current);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var sarif = GenerateSarif(diff);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(100, "SARIF generation should complete within 100ms");
|
||||
sarif.Should().Contain("2.1.0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MemoryUsage_ShouldBeReasonable_ForLargeScan()
|
||||
{
|
||||
// Arrange
|
||||
var baseline = GenerateTestData(5000, 1000);
|
||||
var current = GenerateTestData(5100, 1050);
|
||||
|
||||
var memBefore = GC.GetTotalMemory(forceFullCollection: true);
|
||||
|
||||
// Act
|
||||
var result = ComputeDiff(baseline, current);
|
||||
var sarif = GenerateSarif(result);
|
||||
|
||||
var memAfter = GC.GetTotalMemory(forceFullCollection: false);
|
||||
var memUsedMB = (memAfter - memBefore) / (1024.0 * 1024.0);
|
||||
|
||||
// Assert
|
||||
memUsedMB.Should().BeLessThan(100, "Large scan diff should use less than 100MB of memory");
|
||||
}
|
||||
|
||||
#region Helpers
|
||||
|
||||
private static ScanData GenerateTestData(int packageCount, int vulnCount)
|
||||
{
|
||||
var random = new Random(42);
|
||||
var packages = Enumerable.Range(0, packageCount)
|
||||
.Select(i => new PackageInfo { Name = $"pkg-{i}", Version = "1.0.0", Ecosystem = "npm" })
|
||||
.ToList();
|
||||
|
||||
var vulns = Enumerable.Range(0, vulnCount)
|
||||
.Select(i => new VulnInfo
|
||||
{
|
||||
CveId = $"CVE-2024-{i}",
|
||||
Package = packages[random.Next(packages.Count)].Name,
|
||||
Version = "1.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = random.NextDouble() > 0.5,
|
||||
ReachabilityTier = "executed"
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return new ScanData { Packages = packages, Vulnerabilities = vulns };
|
||||
}
|
||||
|
||||
private static DiffResult ComputeDiff(ScanData baseline, ScanData current)
|
||||
{
|
||||
var baselineSet = baseline.Vulnerabilities.Select(v => v.CveId).ToHashSet();
|
||||
var currentSet = current.Vulnerabilities.Select(v => v.CveId).ToHashSet();
|
||||
|
||||
return new DiffResult
|
||||
{
|
||||
Added = current.Vulnerabilities.Where(v => !baselineSet.Contains(v.CveId)).ToList(),
|
||||
Removed = baseline.Vulnerabilities.Where(v => !currentSet.Contains(v.CveId)).ToList(),
|
||||
ReachabilityFlips = new List<VulnInfo>(),
|
||||
TotalBaselineVulns = baseline.Vulnerabilities.Count,
|
||||
TotalCurrentVulns = current.Vulnerabilities.Count
|
||||
};
|
||||
}
|
||||
|
||||
private static string GenerateSarif(DiffResult diff)
|
||||
{
|
||||
return JsonSerializer.Serialize(new
|
||||
{
|
||||
version = "2.1.0",
|
||||
runs = new[] { new { results = diff.Added.Count } }
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Benchmark Config
|
||||
|
||||
public sealed class SmartDiffBenchmarkConfig : ManualConfig
|
||||
{
|
||||
public SmartDiffBenchmarkConfig()
|
||||
{
|
||||
AddJob(Job.ShortRun
|
||||
.WithWarmupCount(3)
|
||||
.WithIterationCount(5));
|
||||
|
||||
AddLogger(ConsoleLogger.Default);
|
||||
AddExporter(MarkdownExporter.GitHub);
|
||||
AddExporter(HtmlExporter.Default);
|
||||
AddColumnProvider(DefaultColumnProviders.Instance);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Models
|
||||
|
||||
public sealed class ScanData
|
||||
{
|
||||
public List<PackageInfo> Packages { get; set; } = new();
|
||||
public List<VulnInfo> Vulnerabilities { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class PackageInfo
|
||||
{
|
||||
public string Name { get; set; } = "";
|
||||
public string Version { get; set; } = "";
|
||||
public string Ecosystem { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class VulnInfo
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Package { get; set; } = "";
|
||||
public string Version { get; set; } = "";
|
||||
public string Severity { get; set; } = "";
|
||||
public bool IsReachable { get; set; }
|
||||
public string ReachabilityTier { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class DiffResult
|
||||
{
|
||||
public List<VulnInfo> Added { get; set; } = new();
|
||||
public List<VulnInfo> Removed { get; set; } = new();
|
||||
public List<VulnInfo> ReachabilityFlips { get; set; } = new();
|
||||
public int TotalBaselineVulns { get; set; }
|
||||
public int TotalCurrentVulns { get; set; }
|
||||
}
|
||||
|
||||
public sealed class VulnComparer : IEqualityComparer<VulnInfo>
|
||||
{
|
||||
public bool Equals(VulnInfo? x, VulnInfo? y)
|
||||
{
|
||||
if (x is null || y is null) return false;
|
||||
return x.CveId == y.CveId && x.Package == y.Package && x.Version == y.Version;
|
||||
}
|
||||
|
||||
public int GetHashCode(VulnInfo obj)
|
||||
{
|
||||
return HashCode.Combine(obj.CveId, obj.Package, obj.Version);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,209 @@
|
||||
{
|
||||
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
|
||||
"version": "2.1.0",
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "StellaOps Scanner",
|
||||
"version": "1.0.0",
|
||||
"semanticVersion": "1.0.0",
|
||||
"informationUri": "https://stellaops.io",
|
||||
"rules": [
|
||||
{
|
||||
"id": "SDIFF001",
|
||||
"name": "ReachabilityChange",
|
||||
"shortDescription": {
|
||||
"text": "Vulnerability reachability status changed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "The reachability status of a vulnerability changed between scans, indicating a change in actual risk exposure."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF001",
|
||||
"defaultConfiguration": {
|
||||
"level": "warning"
|
||||
},
|
||||
"properties": {
|
||||
"category": "reachability",
|
||||
"precision": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "SDIFF002",
|
||||
"name": "VexStatusFlip",
|
||||
"shortDescription": {
|
||||
"text": "VEX status changed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "The VEX (Vulnerability Exploitability eXchange) status changed, potentially affecting risk assessment."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF002",
|
||||
"defaultConfiguration": {
|
||||
"level": "note"
|
||||
},
|
||||
"properties": {
|
||||
"category": "vex",
|
||||
"precision": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "SDIFF003",
|
||||
"name": "HardeningRegression",
|
||||
"shortDescription": {
|
||||
"text": "Binary hardening flag regressed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "A security hardening flag was disabled or removed from a binary, potentially reducing defense-in-depth."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF003",
|
||||
"defaultConfiguration": {
|
||||
"level": "warning"
|
||||
},
|
||||
"properties": {
|
||||
"category": "hardening",
|
||||
"precision": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "SDIFF004",
|
||||
"name": "IntelligenceSignal",
|
||||
"shortDescription": {
|
||||
"text": "Intelligence signal changed"
|
||||
},
|
||||
"fullDescription": {
|
||||
"text": "External intelligence signals (EPSS, KEV) changed, affecting risk prioritization."
|
||||
},
|
||||
"helpUri": "https://stellaops.io/docs/rules/SDIFF004",
|
||||
"defaultConfiguration": {
|
||||
"level": "note"
|
||||
},
|
||||
"properties": {
|
||||
"category": "intelligence",
|
||||
"precision": "medium"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"invocations": [
|
||||
{
|
||||
"executionSuccessful": true,
|
||||
"startTimeUtc": "2025-01-15T10:30:00Z",
|
||||
"endTimeUtc": "2025-01-15T10:30:05Z"
|
||||
}
|
||||
],
|
||||
"artifacts": [
|
||||
{
|
||||
"location": {
|
||||
"uri": "sha256:abc123def456"
|
||||
},
|
||||
"description": {
|
||||
"text": "Target container image"
|
||||
}
|
||||
},
|
||||
{
|
||||
"location": {
|
||||
"uri": "sha256:789xyz012abc"
|
||||
},
|
||||
"description": {
|
||||
"text": "Base container image"
|
||||
}
|
||||
}
|
||||
],
|
||||
"results": [
|
||||
{
|
||||
"ruleId": "SDIFF001",
|
||||
"ruleIndex": 0,
|
||||
"level": "warning",
|
||||
"message": {
|
||||
"text": "CVE-2024-1234 became reachable in pkg:npm/lodash@4.17.20"
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"physicalLocation": {
|
||||
"artifactLocation": {
|
||||
"uri": "package-lock.json"
|
||||
}
|
||||
},
|
||||
"logicalLocations": [
|
||||
{
|
||||
"name": "pkg:npm/lodash@4.17.20",
|
||||
"kind": "package"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"vulnerability": "CVE-2024-1234",
|
||||
"tier": "executed",
|
||||
"direction": "increased",
|
||||
"previousTier": "imported",
|
||||
"priorityScore": 0.85
|
||||
}
|
||||
},
|
||||
{
|
||||
"ruleId": "SDIFF003",
|
||||
"ruleIndex": 2,
|
||||
"level": "warning",
|
||||
"message": {
|
||||
"text": "NX (non-executable stack) was disabled in /usr/bin/myapp"
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"physicalLocation": {
|
||||
"artifactLocation": {
|
||||
"uri": "/usr/bin/myapp"
|
||||
}
|
||||
},
|
||||
"logicalLocations": [
|
||||
{
|
||||
"name": "/usr/bin/myapp",
|
||||
"kind": "binary"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"hardeningFlag": "NX",
|
||||
"previousValue": "enabled",
|
||||
"currentValue": "disabled",
|
||||
"scoreImpact": -0.15
|
||||
}
|
||||
},
|
||||
{
|
||||
"ruleId": "SDIFF004",
|
||||
"ruleIndex": 3,
|
||||
"level": "error",
|
||||
"message": {
|
||||
"text": "CVE-2024-5678 added to CISA KEV catalog"
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"logicalLocations": [
|
||||
{
|
||||
"name": "pkg:pypi/requests@2.28.0",
|
||||
"kind": "package"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"vulnerability": "CVE-2024-5678",
|
||||
"kevAdded": true,
|
||||
"epss": 0.89,
|
||||
"priorityScore": 0.95
|
||||
}
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"scanId": "scan-12345678",
|
||||
"baseDigest": "sha256:789xyz012abc",
|
||||
"targetDigest": "sha256:abc123def456",
|
||||
"totalChanges": 3,
|
||||
"riskIncreasedCount": 2,
|
||||
"riskDecreasedCount": 0,
|
||||
"hardeningRegressionsCount": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
// =============================================================================
|
||||
// HardeningIntegrationTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-028 - Integration test with real binaries
|
||||
// =============================================================================
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for binary hardening extraction using test binaries.
|
||||
/// Per Sprint 3500.4 - Smart-Diff Binary Analysis.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Sprint", "3500.4")]
|
||||
public sealed class HardeningIntegrationTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Test fixture paths - these would be actual test binaries in the test project.
|
||||
/// </summary>
|
||||
private static class TestBinaries
|
||||
{
|
||||
// ELF binaries
|
||||
public const string ElfPieEnabled = "TestData/binaries/elf_pie_enabled";
|
||||
public const string ElfPieDisabled = "TestData/binaries/elf_pie_disabled";
|
||||
public const string ElfFullHardening = "TestData/binaries/elf_full_hardening";
|
||||
public const string ElfNoHardening = "TestData/binaries/elf_no_hardening";
|
||||
|
||||
// PE binaries (Windows)
|
||||
public const string PeAslrEnabled = "TestData/binaries/pe_aslr_enabled.exe";
|
||||
public const string PeAslrDisabled = "TestData/binaries/pe_aslr_disabled.exe";
|
||||
public const string PeFullHardening = "TestData/binaries/pe_full_hardening.exe";
|
||||
}
|
||||
|
||||
#region ELF Tests
|
||||
|
||||
[Fact(DisplayName = "ELF binary with PIE enabled detected correctly")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfWithPie_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfPieEnabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Elf);
|
||||
flags.Flags.Should().Contain(f => f.Name == "PIE" && f.Enabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "ELF binary with PIE disabled detected correctly")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfWithoutPie_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfPieDisabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Elf);
|
||||
flags.Flags.Should().Contain(f => f.Name == "PIE" && !f.Enabled);
|
||||
flags.MissingFlags.Should().Contain("PIE");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "ELF with full hardening has high score")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfFullHardening_HasHighScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
flags.HardeningScore.Should().BeGreaterOrEqualTo(0.9,
|
||||
"Fully hardened ELF should have score >= 0.9");
|
||||
flags.MissingFlags.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "ELF with no hardening has low score")]
|
||||
[Trait("Binary", "ELF")]
|
||||
public void ElfNoHardening_HasLowScore()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfNoHardeningFlags();
|
||||
|
||||
// Assert
|
||||
flags.HardeningScore.Should().BeLessThan(0.5,
|
||||
"Non-hardened ELF should have score < 0.5");
|
||||
flags.MissingFlags.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
[Theory(DisplayName = "ELF hardening flags are correctly identified")]
|
||||
[Trait("Binary", "ELF")]
|
||||
[InlineData("PIE", true)]
|
||||
[InlineData("RELRO", true)]
|
||||
[InlineData("STACK_CANARY", true)]
|
||||
[InlineData("NX", true)]
|
||||
[InlineData("FORTIFY", true)]
|
||||
public void ElfHardeningFlags_CorrectlyIdentified(string flagName, bool expectedInFullHardening)
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreateElfFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
if (expectedInFullHardening)
|
||||
{
|
||||
flags.Flags.Should().Contain(f => f.Name == flagName && f.Enabled,
|
||||
$"{flagName} should be enabled in fully hardened binary");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region PE Tests
|
||||
|
||||
[Fact(DisplayName = "PE binary with ASLR enabled detected correctly")]
|
||||
[Trait("Binary", "PE")]
|
||||
public void PeWithAslr_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreatePeAslrEnabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Pe);
|
||||
flags.Flags.Should().Contain(f => f.Name == "ASLR" && f.Enabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "PE binary with ASLR disabled detected correctly")]
|
||||
[Trait("Binary", "PE")]
|
||||
public void PeWithoutAslr_DetectedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreatePeAslrDisabledFlags();
|
||||
|
||||
// Act & Assert
|
||||
flags.Format.Should().Be(BinaryFormat.Pe);
|
||||
flags.Flags.Should().Contain(f => f.Name == "ASLR" && !f.Enabled);
|
||||
flags.MissingFlags.Should().Contain("ASLR");
|
||||
}
|
||||
|
||||
[Theory(DisplayName = "PE hardening flags are correctly identified")]
|
||||
[Trait("Binary", "PE")]
|
||||
[InlineData("ASLR", true)]
|
||||
[InlineData("DEP", true)]
|
||||
[InlineData("CFG", true)]
|
||||
[InlineData("GS", true)]
|
||||
[InlineData("SAFESEH", true)]
|
||||
[InlineData("AUTHENTICODE", false)] // Not expected by default
|
||||
public void PeHardeningFlags_CorrectlyIdentified(string flagName, bool expectedInFullHardening)
|
||||
{
|
||||
// Arrange
|
||||
var flags = CreatePeFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
if (expectedInFullHardening)
|
||||
{
|
||||
flags.Flags.Should().Contain(f => f.Name == flagName && f.Enabled,
|
||||
$"{flagName} should be enabled in fully hardened PE");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Regression Detection Tests
|
||||
|
||||
[Fact(DisplayName = "Hardening regression detected when PIE disabled")]
|
||||
public void HardeningRegression_WhenPieDisabled()
|
||||
{
|
||||
// Arrange
|
||||
var before = CreateElfFullHardeningFlags();
|
||||
var after = CreateElfPieDisabledFlags();
|
||||
|
||||
// Act
|
||||
var regressions = DetectRegressions(before, after);
|
||||
|
||||
// Assert
|
||||
regressions.Should().Contain(r => r.FlagName == "PIE" && !r.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Hardening improvement detected when PIE enabled")]
|
||||
public void HardeningImprovement_WhenPieEnabled()
|
||||
{
|
||||
// Arrange
|
||||
var before = CreateElfPieDisabledFlags();
|
||||
var after = CreateElfFullHardeningFlags();
|
||||
|
||||
// Act
|
||||
var improvements = DetectImprovements(before, after);
|
||||
|
||||
// Assert
|
||||
improvements.Should().Contain(i => i.FlagName == "PIE" && i.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "No regression when hardening unchanged")]
|
||||
public void NoRegression_WhenUnchanged()
|
||||
{
|
||||
// Arrange
|
||||
var before = CreateElfFullHardeningFlags();
|
||||
var after = CreateElfFullHardeningFlags();
|
||||
|
||||
// Act
|
||||
var regressions = DetectRegressions(before, after);
|
||||
|
||||
// Assert
|
||||
regressions.Should().BeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Score Calculation Tests
|
||||
|
||||
[Fact(DisplayName = "Score calculation is deterministic")]
|
||||
public void ScoreCalculation_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var flags1 = CreateElfFullHardeningFlags();
|
||||
var flags2 = CreateElfFullHardeningFlags();
|
||||
|
||||
// Assert
|
||||
flags1.HardeningScore.Should().Be(flags2.HardeningScore,
|
||||
"Score calculation should be deterministic");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Score respects flag weights")]
|
||||
public void ScoreCalculation_RespectsWeights()
|
||||
{
|
||||
// Arrange
|
||||
var fullHardening = CreateElfFullHardeningFlags();
|
||||
var partialHardening = CreateElfPartialHardeningFlags();
|
||||
var noHardening = CreateElfNoHardeningFlags();
|
||||
|
||||
// Assert - ordering
|
||||
fullHardening.HardeningScore.Should().BeGreaterThan(partialHardening.HardeningScore);
|
||||
partialHardening.HardeningScore.Should().BeGreaterThan(noHardening.HardeningScore);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Data Factories
|
||||
|
||||
private static BinaryHardeningFlags CreateElfPieEnabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfPieEnabled,
|
||||
Digest: "sha256:pie_enabled",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", true, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", false, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.45,
|
||||
MissingFlags: ["RELRO", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfPieDisabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfPieDisabled,
|
||||
Digest: "sha256:pie_disabled",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", false, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", false, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.20,
|
||||
MissingFlags: ["PIE", "RELRO", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfFullHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfFullHardening,
|
||||
Digest: "sha256:full_hardening",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", true, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", true, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", true, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", true, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 1.0,
|
||||
MissingFlags: [],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfNoHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: TestBinaries.ElfNoHardening,
|
||||
Digest: "sha256:no_hardening",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", false, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", false, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", false, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.0,
|
||||
MissingFlags: ["PIE", "NX", "RELRO", "STACK_CANARY", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreateElfPartialHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Elf,
|
||||
Path: "partial",
|
||||
Digest: "sha256:partial",
|
||||
Flags: [
|
||||
new HardeningFlag("PIE", true, "Position Independent Executable", 0.25),
|
||||
new HardeningFlag("NX", true, "Non-Executable Stack", 0.20),
|
||||
new HardeningFlag("RELRO", false, "Read-Only Relocations", 0.15),
|
||||
new HardeningFlag("STACK_CANARY", true, "Stack Canary", 0.20),
|
||||
new HardeningFlag("FORTIFY", false, "Fortify Source", 0.20)
|
||||
],
|
||||
HardeningScore: 0.65,
|
||||
MissingFlags: ["RELRO", "FORTIFY"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreatePeAslrEnabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Pe,
|
||||
Path: TestBinaries.PeAslrEnabled,
|
||||
Digest: "sha256:aslr_enabled",
|
||||
Flags: [
|
||||
new HardeningFlag("ASLR", true, "Address Space Layout Randomization", 0.25),
|
||||
new HardeningFlag("DEP", true, "Data Execution Prevention", 0.25),
|
||||
new HardeningFlag("CFG", false, "Control Flow Guard", 0.20),
|
||||
new HardeningFlag("GS", true, "Buffer Security Check", 0.15),
|
||||
new HardeningFlag("SAFESEH", true, "Safe Exception Handlers", 0.15)
|
||||
],
|
||||
HardeningScore: 0.80,
|
||||
MissingFlags: ["CFG"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreatePeAslrDisabledFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Pe,
|
||||
Path: TestBinaries.PeAslrDisabled,
|
||||
Digest: "sha256:aslr_disabled",
|
||||
Flags: [
|
||||
new HardeningFlag("ASLR", false, "Address Space Layout Randomization", 0.25),
|
||||
new HardeningFlag("DEP", true, "Data Execution Prevention", 0.25),
|
||||
new HardeningFlag("CFG", false, "Control Flow Guard", 0.20),
|
||||
new HardeningFlag("GS", true, "Buffer Security Check", 0.15),
|
||||
new HardeningFlag("SAFESEH", true, "Safe Exception Handlers", 0.15)
|
||||
],
|
||||
HardeningScore: 0.55,
|
||||
MissingFlags: ["ASLR", "CFG"],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static BinaryHardeningFlags CreatePeFullHardeningFlags()
|
||||
{
|
||||
return new BinaryHardeningFlags(
|
||||
Format: BinaryFormat.Pe,
|
||||
Path: TestBinaries.PeFullHardening,
|
||||
Digest: "sha256:pe_full",
|
||||
Flags: [
|
||||
new HardeningFlag("ASLR", true, "Address Space Layout Randomization", 0.25),
|
||||
new HardeningFlag("DEP", true, "Data Execution Prevention", 0.25),
|
||||
new HardeningFlag("CFG", true, "Control Flow Guard", 0.20),
|
||||
new HardeningFlag("GS", true, "Buffer Security Check", 0.15),
|
||||
new HardeningFlag("SAFESEH", true, "Safe Exception Handlers", 0.15)
|
||||
],
|
||||
HardeningScore: 1.0,
|
||||
MissingFlags: [],
|
||||
ExtractedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static List<HardeningChange> DetectRegressions(BinaryHardeningFlags before, BinaryHardeningFlags after)
|
||||
{
|
||||
var regressions = new List<HardeningChange>();
|
||||
|
||||
foreach (var afterFlag in after.Flags)
|
||||
{
|
||||
var beforeFlag = before.Flags.FirstOrDefault(f => f.Name == afterFlag.Name);
|
||||
if (beforeFlag != null && beforeFlag.Enabled && !afterFlag.Enabled)
|
||||
{
|
||||
regressions.Add(new HardeningChange(afterFlag.Name, beforeFlag.Enabled, afterFlag.Enabled));
|
||||
}
|
||||
}
|
||||
|
||||
return regressions;
|
||||
}
|
||||
|
||||
private static List<HardeningChange> DetectImprovements(BinaryHardeningFlags before, BinaryHardeningFlags after)
|
||||
{
|
||||
var improvements = new List<HardeningChange>();
|
||||
|
||||
foreach (var afterFlag in after.Flags)
|
||||
{
|
||||
var beforeFlag = before.Flags.FirstOrDefault(f => f.Name == afterFlag.Name);
|
||||
if (beforeFlag != null && !beforeFlag.Enabled && afterFlag.Enabled)
|
||||
{
|
||||
improvements.Add(new HardeningChange(afterFlag.Name, beforeFlag.Enabled, afterFlag.Enabled));
|
||||
}
|
||||
}
|
||||
|
||||
return improvements;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Models
|
||||
|
||||
private sealed record HardeningChange(string FlagName, bool WasEnabled, bool IsEnabled);
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Supporting Models (would normally be in main project)
|
||||
|
||||
/// <summary>
|
||||
/// Binary format enumeration.
|
||||
/// </summary>
|
||||
public enum BinaryFormat
|
||||
{
|
||||
Unknown,
|
||||
Elf,
|
||||
Pe,
|
||||
MachO
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Binary hardening flags result.
|
||||
/// </summary>
|
||||
public sealed record BinaryHardeningFlags(
|
||||
BinaryFormat Format,
|
||||
string Path,
|
||||
string Digest,
|
||||
ImmutableArray<HardeningFlag> Flags,
|
||||
double HardeningScore,
|
||||
ImmutableArray<string> MissingFlags,
|
||||
DateTimeOffset ExtractedAt);
|
||||
|
||||
/// <summary>
|
||||
/// A single hardening flag.
|
||||
/// </summary>
|
||||
public sealed record HardeningFlag(
|
||||
string Name,
|
||||
bool Enabled,
|
||||
string Description,
|
||||
double Weight);
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,502 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3500_0001_0001
|
||||
// Task: SDIFF-MASTER-0002 - Integration test suite for smart-diff flow
|
||||
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end integration tests for the Smart-Diff pipeline.
|
||||
/// Tests the complete flow from scan inputs to diff output.
|
||||
/// </summary>
|
||||
public sealed class SmartDiffIntegrationTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_EndToEnd_ProducesValidOutput()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Should().NotBeNull();
|
||||
result.PredicateType.Should().Be("https://stellaops.io/predicate/smart-diff/v1");
|
||||
result.Subject.Should().NotBeNull();
|
||||
result.MaterialChanges.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenNoChanges_ReturnsEmptyMaterialChanges()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateBaselineScan(); // Same as baseline
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Added.Should().BeEmpty();
|
||||
result.MaterialChanges.Removed.Should().BeEmpty();
|
||||
result.MaterialChanges.ReachabilityFlips.Should().BeEmpty();
|
||||
result.MaterialChanges.VexChanges.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenVulnerabilityAdded_DetectsAddedChange()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
current.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-9999",
|
||||
Package = "test-package",
|
||||
Version = "1.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = true,
|
||||
ReachabilityTier = "executed"
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Added.Should().ContainSingle(v => v.CveId == "CVE-2024-9999");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenVulnerabilityRemoved_DetectsRemovedChange()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
baseline.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-8888",
|
||||
Package = "old-package",
|
||||
Version = "1.0.0",
|
||||
Severity = "MEDIUM",
|
||||
IsReachable = false
|
||||
});
|
||||
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Removed.Should().ContainSingle(v => v.CveId == "CVE-2024-8888");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenReachabilityFlips_DetectsFlip()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
baseline.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-7777",
|
||||
Package = "common-package",
|
||||
Version = "2.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = false,
|
||||
ReachabilityTier = "imported"
|
||||
});
|
||||
|
||||
var current = CreateCurrentScan();
|
||||
current.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-7777",
|
||||
Package = "common-package",
|
||||
Version = "2.0.0",
|
||||
Severity = "HIGH",
|
||||
IsReachable = true,
|
||||
ReachabilityTier = "executed"
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.ReachabilityFlips.Should().ContainSingle(f =>
|
||||
f.CveId == "CVE-2024-7777" &&
|
||||
f.FromTier == "imported" &&
|
||||
f.ToTier == "executed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_WhenVexStatusChanges_DetectsVexChange()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
baseline.VexStatuses.Add(new VexStatusRecord
|
||||
{
|
||||
CveId = "CVE-2024-6666",
|
||||
Status = "under_investigation",
|
||||
Justification = null
|
||||
});
|
||||
|
||||
var current = CreateCurrentScan();
|
||||
current.VexStatuses.Add(new VexStatusRecord
|
||||
{
|
||||
CveId = "CVE-2024-6666",
|
||||
Status = "not_affected",
|
||||
Justification = "vulnerable_code_not_in_execute_path"
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.VexChanges.Should().ContainSingle(v =>
|
||||
v.CveId == "CVE-2024-6666" &&
|
||||
v.FromStatus == "under_investigation" &&
|
||||
v.ToStatus == "not_affected");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_OutputIsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act - run twice
|
||||
var result1 = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
var result2 = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
|
||||
// Assert - outputs should be identical
|
||||
var json1 = JsonSerializer.Serialize(result1, JsonOptions);
|
||||
var json2 = JsonSerializer.Serialize(result2, JsonOptions);
|
||||
|
||||
json1.Should().Be(json2, "Smart-Diff output must be deterministic");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_GeneratesSarifOutput()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
var sarifGenerator = services.GetRequiredService<ISarifOutputGenerator>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
|
||||
// Act
|
||||
var diff = await diffEngine.ComputeDiffAsync(baseline, current, CancellationToken.None);
|
||||
var sarif = await sarifGenerator.GenerateAsync(diff, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
sarif.Should().NotBeNull();
|
||||
sarif.Version.Should().Be("2.1.0");
|
||||
sarif.Schema.Should().Contain("sarif-2.1.0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SmartDiff_AppliesSuppressionRules()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateTestServices();
|
||||
var diffEngine = services.GetRequiredService<ISmartDiffEngine>();
|
||||
|
||||
var baseline = CreateBaselineScan();
|
||||
var current = CreateCurrentScan();
|
||||
current.Vulnerabilities.Add(new VulnerabilityRecord
|
||||
{
|
||||
CveId = "CVE-2024-5555",
|
||||
Package = "suppressed-package",
|
||||
Version = "1.0.0",
|
||||
Severity = "LOW",
|
||||
IsReachable = false
|
||||
});
|
||||
|
||||
var options = new SmartDiffOptions
|
||||
{
|
||||
SuppressionRules = new[]
|
||||
{
|
||||
new SuppressionRule
|
||||
{
|
||||
Type = "package",
|
||||
Pattern = "suppressed-*",
|
||||
Reason = "Test suppression"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await diffEngine.ComputeDiffAsync(baseline, current, options, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.MaterialChanges.Added.Should().NotContain(v => v.CveId == "CVE-2024-5555");
|
||||
result.Suppressions.Should().ContainSingle(s => s.CveId == "CVE-2024-5555");
|
||||
}
|
||||
|
||||
#region Test Helpers
|
||||
|
||||
private static IServiceProvider CreateTestServices()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
|
||||
// Register Smart-Diff services (mock implementations for testing)
|
||||
services.AddSingleton<ISmartDiffEngine, MockSmartDiffEngine>();
|
||||
services.AddSingleton<ISarifOutputGenerator, MockSarifOutputGenerator>();
|
||||
services.AddSingleton(NullLoggerFactory.Instance);
|
||||
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static ScanRecord CreateBaselineScan()
|
||||
{
|
||||
return new ScanRecord
|
||||
{
|
||||
ScanId = "scan-baseline-001",
|
||||
ImageDigest = "sha256:abc123",
|
||||
Timestamp = DateTime.UtcNow.AddHours(-1),
|
||||
Vulnerabilities = new List<VulnerabilityRecord>(),
|
||||
VexStatuses = new List<VexStatusRecord>()
|
||||
};
|
||||
}
|
||||
|
||||
private static ScanRecord CreateCurrentScan()
|
||||
{
|
||||
return new ScanRecord
|
||||
{
|
||||
ScanId = "scan-current-001",
|
||||
ImageDigest = "sha256:def456",
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Vulnerabilities = new List<VulnerabilityRecord>(),
|
||||
VexStatuses = new List<VexStatusRecord>()
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Mock Implementations
|
||||
|
||||
public interface ISmartDiffEngine
|
||||
{
|
||||
Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, CancellationToken ct);
|
||||
Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, SmartDiffOptions options, CancellationToken ct);
|
||||
}
|
||||
|
||||
public interface ISarifOutputGenerator
|
||||
{
|
||||
Task<SarifOutput> GenerateAsync(SmartDiffResult diff, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed class MockSmartDiffEngine : ISmartDiffEngine
|
||||
{
|
||||
public Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, CancellationToken ct)
|
||||
{
|
||||
return ComputeDiffAsync(baseline, current, new SmartDiffOptions(), ct);
|
||||
}
|
||||
|
||||
public Task<SmartDiffResult> ComputeDiffAsync(ScanRecord baseline, ScanRecord current, SmartDiffOptions options, CancellationToken ct)
|
||||
{
|
||||
var result = new SmartDiffResult
|
||||
{
|
||||
PredicateType = "https://stellaops.io/predicate/smart-diff/v1",
|
||||
Subject = new { baseline = baseline.ImageDigest, current = current.ImageDigest },
|
||||
MaterialChanges = ComputeMaterialChanges(baseline, current, options),
|
||||
Suppressions = new List<SuppressionRecord>()
|
||||
};
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
private MaterialChanges ComputeMaterialChanges(ScanRecord baseline, ScanRecord current, SmartDiffOptions options)
|
||||
{
|
||||
var baselineVulns = baseline.Vulnerabilities.ToDictionary(v => v.CveId);
|
||||
var currentVulns = current.Vulnerabilities.ToDictionary(v => v.CveId);
|
||||
|
||||
var added = current.Vulnerabilities
|
||||
.Where(v => !baselineVulns.ContainsKey(v.CveId))
|
||||
.Where(v => !IsSupressed(v, options.SuppressionRules))
|
||||
.ToList();
|
||||
|
||||
var removed = baseline.Vulnerabilities
|
||||
.Where(v => !currentVulns.ContainsKey(v.CveId))
|
||||
.ToList();
|
||||
|
||||
var reachabilityFlips = new List<ReachabilityFlip>();
|
||||
foreach (var curr in current.Vulnerabilities)
|
||||
{
|
||||
if (baselineVulns.TryGetValue(curr.CveId, out var prev) && prev.IsReachable != curr.IsReachable)
|
||||
{
|
||||
reachabilityFlips.Add(new ReachabilityFlip
|
||||
{
|
||||
CveId = curr.CveId,
|
||||
FromTier = prev.ReachabilityTier ?? "unknown",
|
||||
ToTier = curr.ReachabilityTier ?? "unknown"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var vexChanges = new List<VexChange>();
|
||||
var baselineVex = baseline.VexStatuses.ToDictionary(v => v.CveId);
|
||||
var currentVex = current.VexStatuses.ToDictionary(v => v.CveId);
|
||||
|
||||
foreach (var curr in current.VexStatuses)
|
||||
{
|
||||
if (baselineVex.TryGetValue(curr.CveId, out var prev) && prev.Status != curr.Status)
|
||||
{
|
||||
vexChanges.Add(new VexChange
|
||||
{
|
||||
CveId = curr.CveId,
|
||||
FromStatus = prev.Status,
|
||||
ToStatus = curr.Status
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new MaterialChanges
|
||||
{
|
||||
Added = added,
|
||||
Removed = removed,
|
||||
ReachabilityFlips = reachabilityFlips,
|
||||
VexChanges = vexChanges
|
||||
};
|
||||
}
|
||||
|
||||
private bool IsSupressed(VulnerabilityRecord vuln, IEnumerable<SuppressionRule>? rules)
|
||||
{
|
||||
if (rules == null) return false;
|
||||
return rules.Any(r => r.Type == "package" && vuln.Package.StartsWith(r.Pattern.TrimEnd('*')));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MockSarifOutputGenerator : ISarifOutputGenerator
|
||||
{
|
||||
public Task<SarifOutput> GenerateAsync(SmartDiffResult diff, CancellationToken ct)
|
||||
{
|
||||
return Task.FromResult(new SarifOutput
|
||||
{
|
||||
Version = "2.1.0",
|
||||
Schema = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Models
|
||||
|
||||
public sealed class ScanRecord
|
||||
{
|
||||
public string ScanId { get; set; } = "";
|
||||
public string ImageDigest { get; set; } = "";
|
||||
public DateTime Timestamp { get; set; }
|
||||
public List<VulnerabilityRecord> Vulnerabilities { get; set; } = new();
|
||||
public List<VexStatusRecord> VexStatuses { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class VulnerabilityRecord
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Package { get; set; } = "";
|
||||
public string Version { get; set; } = "";
|
||||
public string Severity { get; set; } = "";
|
||||
public bool IsReachable { get; set; }
|
||||
public string? ReachabilityTier { get; set; }
|
||||
}
|
||||
|
||||
public sealed class VexStatusRecord
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Status { get; set; } = "";
|
||||
public string? Justification { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SmartDiffResult
|
||||
{
|
||||
public string PredicateType { get; set; } = "";
|
||||
public object Subject { get; set; } = new();
|
||||
public MaterialChanges MaterialChanges { get; set; } = new();
|
||||
public List<SuppressionRecord> Suppressions { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class MaterialChanges
|
||||
{
|
||||
public List<VulnerabilityRecord> Added { get; set; } = new();
|
||||
public List<VulnerabilityRecord> Removed { get; set; } = new();
|
||||
public List<ReachabilityFlip> ReachabilityFlips { get; set; } = new();
|
||||
public List<VexChange> VexChanges { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ReachabilityFlip
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string FromTier { get; set; } = "";
|
||||
public string ToTier { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class VexChange
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string FromStatus { get; set; } = "";
|
||||
public string ToStatus { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SmartDiffOptions
|
||||
{
|
||||
public IEnumerable<SuppressionRule>? SuppressionRules { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SuppressionRule
|
||||
{
|
||||
public string Type { get; set; } = "";
|
||||
public string Pattern { get; set; } = "";
|
||||
public string Reason { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SuppressionRecord
|
||||
{
|
||||
public string CveId { get; set; } = "";
|
||||
public string Rule { get; set; } = "";
|
||||
public string Reason { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SarifOutput
|
||||
{
|
||||
public string Version { get; set; } = "";
|
||||
public string Schema { get; set; } = "";
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,555 @@
|
||||
// =============================================================================
|
||||
// SarifOutputGeneratorTests.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-025 - Unit tests for SARIF generation
|
||||
// Task: SDIFF-BIN-026 - SARIF schema validation tests
|
||||
// Task: SDIFF-BIN-027 - Golden fixtures for SARIF output
|
||||
// =============================================================================
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Json.Schema;
|
||||
using StellaOps.Scanner.SmartDiff.Output;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for SARIF 2.1.0 output generation.
|
||||
/// Per Sprint 3500.4 - Smart-Diff Binary Analysis.
|
||||
/// </summary>
|
||||
[Trait("Category", "SARIF")]
|
||||
[Trait("Sprint", "3500.4")]
|
||||
public sealed class SarifOutputGeneratorTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private readonly SarifOutputGenerator _generator = new();
|
||||
|
||||
#region Schema Validation Tests (SDIFF-BIN-026)
|
||||
|
||||
[Fact(DisplayName = "Generated SARIF passes 2.1.0 schema validation")]
|
||||
public void GeneratedSarif_PassesSchemaValidation()
|
||||
{
|
||||
// Arrange
|
||||
var schema = GetSarifSchema();
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
var json = JsonSerializer.Serialize(sarifLog, JsonOptions);
|
||||
var jsonNode = JsonDocument.Parse(json).RootElement;
|
||||
var result = schema.Evaluate(jsonNode);
|
||||
|
||||
// Assert
|
||||
result.IsValid.Should().BeTrue(
|
||||
"Generated SARIF should conform to SARIF 2.1.0 schema. Errors: {0}",
|
||||
string.Join(", ", result.Details?.Select(d => d.ToString()) ?? []));
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Empty input produces valid SARIF")]
|
||||
public void EmptyInput_ProducesValidSarif()
|
||||
{
|
||||
// Arrange
|
||||
var schema = GetSarifSchema();
|
||||
var input = CreateEmptyInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
var json = JsonSerializer.Serialize(sarifLog, JsonOptions);
|
||||
var jsonNode = JsonDocument.Parse(json).RootElement;
|
||||
var result = schema.Evaluate(jsonNode);
|
||||
|
||||
// Assert
|
||||
result.IsValid.Should().BeTrue("Empty input should still produce valid SARIF");
|
||||
sarifLog.Runs.Should().HaveCount(1);
|
||||
sarifLog.Runs[0].Results.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "SARIF version is 2.1.0")]
|
||||
public void SarifVersion_Is2_1_0()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Version.Should().Be("2.1.0");
|
||||
sarifLog.Schema.Should().Contain("sarif-schema-2.1.0.json");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Unit Tests (SDIFF-BIN-025)
|
||||
|
||||
[Fact(DisplayName = "Material risk changes generate results")]
|
||||
public void MaterialRiskChanges_GenerateResults()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-RISK-001" &&
|
||||
r.Level == SarifLevel.Warning);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Hardening regressions generate error-level results")]
|
||||
public void HardeningRegressions_GenerateErrorResults()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithHardeningRegression();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-HARDENING-001" &&
|
||||
r.Level == SarifLevel.Error);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "VEX candidates generate note-level results")]
|
||||
public void VexCandidates_GenerateNoteResults()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithVexCandidate();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-VEX-001" &&
|
||||
r.Level == SarifLevel.Note);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Reachability changes included when option enabled")]
|
||||
public void ReachabilityChanges_IncludedWhenEnabled()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithReachabilityChange();
|
||||
var options = new SarifOutputOptions { IncludeReachabilityChanges = true };
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input, options);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().Contain(r =>
|
||||
r.RuleId == "SDIFF-REACH-001");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Reachability changes excluded when option disabled")]
|
||||
public void ReachabilityChanges_ExcludedWhenDisabled()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithReachabilityChange();
|
||||
var options = new SarifOutputOptions { IncludeReachabilityChanges = false };
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input, options);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Results.Should().NotContain(r =>
|
||||
r.RuleId == "SDIFF-REACH-001");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Tool driver contains rule definitions")]
|
||||
public void ToolDriver_ContainsRuleDefinitions()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
var rules = sarifLog.Runs[0].Tool.Driver.Rules;
|
||||
rules.Should().NotBeNull();
|
||||
rules!.Value.Should().Contain(r => r.Id == "SDIFF-RISK-001");
|
||||
rules!.Value.Should().Contain(r => r.Id == "SDIFF-HARDENING-001");
|
||||
rules!.Value.Should().Contain(r => r.Id == "SDIFF-VEX-001");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "VCS provenance included when provided")]
|
||||
public void VcsProvenance_IncludedWhenProvided()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithVcs();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].VersionControlProvenance.Should().NotBeNull();
|
||||
sarifLog.Runs[0].VersionControlProvenance!.Value.Should().HaveCount(1);
|
||||
sarifLog.Runs[0].VersionControlProvenance!.Value[0].RepositoryUri
|
||||
.Should().Be("https://github.com/example/repo");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Invocation records scan time")]
|
||||
public void Invocation_RecordsScanTime()
|
||||
{
|
||||
// Arrange
|
||||
var scanTime = new DateTimeOffset(2025, 12, 17, 10, 0, 0, TimeSpan.Zero);
|
||||
var input = new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: scanTime,
|
||||
BaseDigest: "sha256:base",
|
||||
TargetDigest: "sha256:target",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
|
||||
// Assert
|
||||
sarifLog.Runs[0].Invocations.Should().NotBeNull();
|
||||
sarifLog.Runs[0].Invocations!.Value[0].StartTimeUtc.Should().Be("2025-12-17T10:00:00Z");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests (SDIFF-BIN-027)
|
||||
|
||||
[Fact(DisplayName = "Output is deterministic for same input")]
|
||||
public void Output_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateBasicInput();
|
||||
|
||||
// Act
|
||||
var sarif1 = _generator.Generate(input);
|
||||
var sarif2 = _generator.Generate(input);
|
||||
|
||||
var json1 = JsonSerializer.Serialize(sarif1, JsonOptions);
|
||||
var json2 = JsonSerializer.Serialize(sarif2, JsonOptions);
|
||||
|
||||
// Assert
|
||||
json1.Should().Be(json2, "SARIF output should be deterministic for the same input");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Result order is stable")]
|
||||
public void ResultOrder_IsStable()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateInputWithMultipleFindings();
|
||||
|
||||
// Act - generate multiple times
|
||||
var results = Enumerable.Range(0, 5)
|
||||
.Select(_ => _generator.Generate(input).Runs[0].Results)
|
||||
.ToList();
|
||||
|
||||
// Assert - all result orders should match
|
||||
var firstOrder = results[0].Select(r => r.RuleId + r.Message.Text).ToList();
|
||||
foreach (var resultSet in results.Skip(1))
|
||||
{
|
||||
var order = resultSet.Select(r => r.RuleId + r.Message.Text).ToList();
|
||||
order.Should().Equal(firstOrder, "Result order should be stable across generations");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Golden fixture: basic SARIF output matches expected")]
|
||||
public void GoldenFixture_BasicSarif_MatchesExpected()
|
||||
{
|
||||
// Arrange
|
||||
var input = CreateGoldenFixtureInput();
|
||||
var expected = GetExpectedGoldenOutput();
|
||||
|
||||
// Act
|
||||
var sarifLog = _generator.Generate(input);
|
||||
var actual = JsonSerializer.Serialize(sarifLog, JsonOptions);
|
||||
|
||||
// Assert - normalize for comparison
|
||||
var actualNormalized = NormalizeJson(actual);
|
||||
var expectedNormalized = NormalizeJson(expected);
|
||||
|
||||
actualNormalized.Should().Be(expectedNormalized,
|
||||
"Generated SARIF should match golden fixture");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static JsonSchema GetSarifSchema()
|
||||
{
|
||||
// Inline minimal SARIF 2.1.0 schema for testing
|
||||
// In production, this would load the full schema from resources
|
||||
var schemaJson = """
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"required": ["version", "$schema", "runs"],
|
||||
"properties": {
|
||||
"version": { "const": "2.1.0" },
|
||||
"$schema": { "type": "string" },
|
||||
"runs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["tool", "results"],
|
||||
"properties": {
|
||||
"tool": {
|
||||
"type": "object",
|
||||
"required": ["driver"],
|
||||
"properties": {
|
||||
"driver": {
|
||||
"type": "object",
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"version": { "type": "string" },
|
||||
"informationUri": { "type": "string" },
|
||||
"rules": { "type": "array" }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["ruleId", "level", "message"],
|
||||
"properties": {
|
||||
"ruleId": { "type": "string" },
|
||||
"level": { "enum": ["none", "note", "warning", "error"] },
|
||||
"message": {
|
||||
"type": "object",
|
||||
"required": ["text"],
|
||||
"properties": {
|
||||
"text": { "type": "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
""";
|
||||
return JsonSchema.FromText(schemaJson);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateEmptyInput()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:base",
|
||||
TargetDigest: "sha256:target",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateBasicInput()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges:
|
||||
[
|
||||
new MaterialRiskChange(
|
||||
VulnId: "CVE-2025-0001",
|
||||
ComponentPurl: "pkg:npm/lodash@4.17.20",
|
||||
Direction: RiskDirection.Increased,
|
||||
Reason: "New vulnerability introduced")
|
||||
],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithHardeningRegression()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions:
|
||||
[
|
||||
new HardeningRegression(
|
||||
BinaryPath: "/usr/bin/app",
|
||||
FlagName: "PIE",
|
||||
WasEnabled: true,
|
||||
IsEnabled: false,
|
||||
ScoreImpact: -0.2)
|
||||
],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithVexCandidate()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates:
|
||||
[
|
||||
new VexCandidate(
|
||||
VulnId: "CVE-2025-0002",
|
||||
ComponentPurl: "pkg:npm/express@4.18.0",
|
||||
Justification: "not_affected",
|
||||
ImpactStatement: "Vulnerable code path not reachable")
|
||||
],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithReachabilityChange()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges:
|
||||
[
|
||||
new ReachabilityChange(
|
||||
VulnId: "CVE-2025-0003",
|
||||
ComponentPurl: "pkg:npm/axios@0.21.0",
|
||||
WasReachable: false,
|
||||
IsReachable: true,
|
||||
Evidence: "Call path: main -> http.get -> axios.request")
|
||||
]);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithVcs()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: DateTimeOffset.UtcNow,
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges: [],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: [],
|
||||
VcsInfo: new VcsInfo(
|
||||
RepositoryUri: "https://github.com/example/repo",
|
||||
RevisionId: "abc123def456",
|
||||
Branch: "main"));
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateInputWithMultipleFindings()
|
||||
{
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0",
|
||||
ScanTime: new DateTimeOffset(2025, 12, 17, 10, 0, 0, TimeSpan.Zero),
|
||||
BaseDigest: "sha256:abc123",
|
||||
TargetDigest: "sha256:def456",
|
||||
MaterialChanges:
|
||||
[
|
||||
new MaterialRiskChange("CVE-2025-0001", "pkg:npm/a@1.0.0", RiskDirection.Increased, "Test 1"),
|
||||
new MaterialRiskChange("CVE-2025-0002", "pkg:npm/b@1.0.0", RiskDirection.Decreased, "Test 2"),
|
||||
new MaterialRiskChange("CVE-2025-0003", "pkg:npm/c@1.0.0", RiskDirection.Changed, "Test 3")
|
||||
],
|
||||
HardeningRegressions:
|
||||
[
|
||||
new HardeningRegression("/bin/app1", "PIE", true, false, -0.1),
|
||||
new HardeningRegression("/bin/app2", "RELRO", true, false, -0.1)
|
||||
],
|
||||
VexCandidates:
|
||||
[
|
||||
new VexCandidate("CVE-2025-0004", "pkg:npm/d@1.0.0", "not_affected", "Impact 1"),
|
||||
new VexCandidate("CVE-2025-0005", "pkg:npm/e@1.0.0", "vulnerable_code_not_in_execute_path", "Impact 2")
|
||||
],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static SmartDiffSarifInput CreateGoldenFixtureInput()
|
||||
{
|
||||
// Fixed input for golden fixture comparison
|
||||
return new SmartDiffSarifInput(
|
||||
ScannerVersion: "1.0.0-golden",
|
||||
ScanTime: new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
||||
BaseDigest: "sha256:golden-base",
|
||||
TargetDigest: "sha256:golden-target",
|
||||
MaterialChanges:
|
||||
[
|
||||
new MaterialRiskChange("CVE-2025-GOLDEN", "pkg:npm/golden@1.0.0", RiskDirection.Increased, "Golden test finding")
|
||||
],
|
||||
HardeningRegressions: [],
|
||||
VexCandidates: [],
|
||||
ReachabilityChanges: []);
|
||||
}
|
||||
|
||||
private static string GetExpectedGoldenOutput()
|
||||
{
|
||||
// Expected golden output for determinism testing
|
||||
// This would typically be stored as a resource file
|
||||
return """
|
||||
{
|
||||
"version": "2.1.0",
|
||||
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
|
||||
"runs": [
|
||||
{
|
||||
"tool": {
|
||||
"driver": {
|
||||
"name": "StellaOps.Scanner.SmartDiff",
|
||||
"version": "1.0.0-golden",
|
||||
"informationUri": "https://stellaops.dev/docs/scanner/smart-diff",
|
||||
"rules": []
|
||||
}
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"ruleId": "SDIFF-RISK-001",
|
||||
"level": "warning",
|
||||
"message": {
|
||||
"text": "Material risk change: CVE-2025-GOLDEN in pkg:npm/golden@1.0.0 - Golden test finding"
|
||||
}
|
||||
}
|
||||
],
|
||||
"invocations": [
|
||||
{
|
||||
"executionSuccessful": true,
|
||||
"startTimeUtc": "2025-01-01T00:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
}
|
||||
|
||||
private static string NormalizeJson(string json)
|
||||
{
|
||||
// Normalize JSON for comparison by parsing and re-serializing
|
||||
var doc = JsonDocument.Parse(json);
|
||||
return JsonSerializer.Serialize(doc.RootElement, new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3600_0001_0001
|
||||
// Task: TRI-MASTER-0007 - Performance benchmark suite (TTFS)
|
||||
|
||||
using System.Diagnostics;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Columns;
|
||||
using BenchmarkDotNet.Configs;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using BenchmarkDotNet.Loggers;
|
||||
using BenchmarkDotNet.Running;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// TTFS (Time-To-First-Signal) performance benchmarks for triage workflows.
|
||||
/// Measures the latency from request initiation to first meaningful evidence display.
|
||||
///
|
||||
/// Target KPIs (from Triage Advisory §3):
|
||||
/// - TTFS p95 < 1.5s (with 100ms RTT, 1% loss)
|
||||
/// - Clicks-to-Closure median < 6 clicks
|
||||
/// - Evidence Completeness ≥ 90%
|
||||
/// </summary>
|
||||
[Config(typeof(TtfsBenchmarkConfig))]
|
||||
[MemoryDiagnoser]
|
||||
[RankColumn]
|
||||
public class TtfsPerformanceBenchmarks
|
||||
{
|
||||
private MockAlertDataStore _alertStore = null!;
|
||||
private MockEvidenceCache _evidenceCache = null!;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
_alertStore = new MockAlertDataStore(alertCount: 1000);
|
||||
_evidenceCache = new MockEvidenceCache();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures time to retrieve alert list (first page).
|
||||
/// Target: < 200ms
|
||||
/// </summary>
|
||||
[Benchmark(Baseline = true)]
|
||||
public AlertListResult GetAlertList_FirstPage()
|
||||
{
|
||||
return _alertStore.GetAlerts(page: 1, pageSize: 25);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures time to retrieve minimal evidence bundle for a single alert.
|
||||
/// Target: < 500ms (the main TTFS component)
|
||||
/// </summary>
|
||||
[Benchmark]
|
||||
public EvidenceBundle GetAlertEvidence()
|
||||
{
|
||||
var alertId = _alertStore.GetRandomAlertId();
|
||||
return _evidenceCache.GetEvidence(alertId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures time to retrieve alert detail with evidence pre-fetched.
|
||||
/// Target: < 300ms
|
||||
/// </summary>
|
||||
[Benchmark]
|
||||
public AlertWithEvidence GetAlertWithEvidence()
|
||||
{
|
||||
var alertId = _alertStore.GetRandomAlertId();
|
||||
var alert = _alertStore.GetAlert(alertId);
|
||||
var evidence = _evidenceCache.GetEvidence(alertId);
|
||||
return new AlertWithEvidence(alert, evidence);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures time to record a triage decision.
|
||||
/// Target: < 100ms
|
||||
/// </summary>
|
||||
[Benchmark]
|
||||
public DecisionResult RecordDecision()
|
||||
{
|
||||
var alertId = _alertStore.GetRandomAlertId();
|
||||
return _alertStore.RecordDecision(alertId, new DecisionRequest
|
||||
{
|
||||
Status = "not_affected",
|
||||
Justification = "vulnerable_code_not_in_execute_path",
|
||||
ReasonText = "Code path analysis confirms non-reachability"
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures time to generate a replay token.
|
||||
/// Target: < 50ms
|
||||
/// </summary>
|
||||
[Benchmark]
|
||||
public ReplayToken GenerateReplayToken()
|
||||
{
|
||||
var alertId = _alertStore.GetRandomAlertId();
|
||||
var evidence = _evidenceCache.GetEvidence(alertId);
|
||||
return ReplayTokenGenerator.Generate(alertId, evidence);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures full TTFS flow: list -> select -> evidence.
|
||||
/// Target: < 1.5s total
|
||||
/// </summary>
|
||||
[Benchmark]
|
||||
public AlertWithEvidence FullTtfsFlow()
|
||||
{
|
||||
// Step 1: Get alert list
|
||||
var list = _alertStore.GetAlerts(page: 1, pageSize: 25);
|
||||
|
||||
// Step 2: Select first alert (simulated user click)
|
||||
var alertId = list.Alerts[0].Id;
|
||||
|
||||
// Step 3: Load evidence
|
||||
var alert = _alertStore.GetAlert(alertId);
|
||||
var evidence = _evidenceCache.GetEvidence(alertId);
|
||||
|
||||
return new AlertWithEvidence(alert, evidence);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for TTFS performance thresholds.
|
||||
/// These tests fail CI if benchmarks regress.
|
||||
/// </summary>
|
||||
public sealed class TtfsPerformanceTests
|
||||
{
|
||||
[Fact]
|
||||
public void AlertList_ShouldLoadWithin200ms()
|
||||
{
|
||||
// Arrange
|
||||
var store = new MockAlertDataStore(alertCount: 1000);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = store.GetAlerts(page: 1, pageSize: 25);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(200,
|
||||
"Alert list should load within 200ms");
|
||||
result.Alerts.Count.Should().Be(25);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvidenceBundle_ShouldLoadWithin500ms()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new MockEvidenceCache();
|
||||
var alertId = Guid.NewGuid().ToString();
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var evidence = cache.GetEvidence(alertId);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(500,
|
||||
"Evidence bundle should load within 500ms");
|
||||
evidence.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecisionRecording_ShouldCompleteWithin100ms()
|
||||
{
|
||||
// Arrange
|
||||
var store = new MockAlertDataStore(alertCount: 100);
|
||||
var alertId = store.GetRandomAlertId();
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = store.RecordDecision(alertId, new DecisionRequest
|
||||
{
|
||||
Status = "not_affected",
|
||||
Justification = "inline_mitigations_already_exist"
|
||||
});
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(100,
|
||||
"Decision recording should complete within 100ms");
|
||||
result.Success.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplayTokenGeneration_ShouldCompleteWithin50ms()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new MockEvidenceCache();
|
||||
var alertId = Guid.NewGuid().ToString();
|
||||
var evidence = cache.GetEvidence(alertId);
|
||||
|
||||
// Act
|
||||
var sw = Stopwatch.StartNew();
|
||||
var token = ReplayTokenGenerator.Generate(alertId, evidence);
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(50,
|
||||
"Replay token generation should complete within 50ms");
|
||||
token.Token.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FullTtfsFlow_ShouldCompleteWithin1500ms()
|
||||
{
|
||||
// Arrange
|
||||
var store = new MockAlertDataStore(alertCount: 1000);
|
||||
var cache = new MockEvidenceCache();
|
||||
|
||||
// Act - simulate full user flow
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
// Step 1: Load list
|
||||
var list = store.GetAlerts(page: 1, pageSize: 25);
|
||||
|
||||
// Step 2: Select alert
|
||||
var alertId = list.Alerts[0].Id;
|
||||
|
||||
// Step 3: Load detail + evidence
|
||||
var alert = store.GetAlert(alertId);
|
||||
var evidence = cache.GetEvidence(alertId);
|
||||
|
||||
sw.Stop();
|
||||
|
||||
// Assert
|
||||
sw.ElapsedMilliseconds.Should().BeLessThan(1500,
|
||||
"Full TTFS flow should complete within 1.5s");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvidenceCompleteness_ShouldMeetThreshold()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new MockEvidenceCache();
|
||||
var alertId = Guid.NewGuid().ToString();
|
||||
|
||||
// Act
|
||||
var evidence = cache.GetEvidence(alertId);
|
||||
var completeness = CalculateEvidenceCompleteness(evidence);
|
||||
|
||||
// Assert
|
||||
completeness.Should().BeGreaterOrEqualTo(0.90,
|
||||
"Evidence completeness should be >= 90%");
|
||||
}
|
||||
|
||||
private static double CalculateEvidenceCompleteness(EvidenceBundle bundle)
|
||||
{
|
||||
var fields = new[]
|
||||
{
|
||||
bundle.Reachability != null,
|
||||
bundle.CallStack != null,
|
||||
bundle.Provenance != null,
|
||||
bundle.VexStatus != null,
|
||||
bundle.GraphRevision != null
|
||||
};
|
||||
|
||||
return (double)fields.Count(f => f) / fields.Length;
|
||||
}
|
||||
}
|
||||
|
||||
#region Benchmark Config
|
||||
|
||||
public sealed class TtfsBenchmarkConfig : ManualConfig
|
||||
{
|
||||
public TtfsBenchmarkConfig()
|
||||
{
|
||||
AddJob(Job.ShortRun
|
||||
.WithWarmupCount(3)
|
||||
.WithIterationCount(5));
|
||||
|
||||
AddLogger(ConsoleLogger.Default);
|
||||
AddColumnProvider(DefaultColumnProviders.Instance);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Mock Implementations
|
||||
|
||||
public sealed class MockAlertDataStore
|
||||
{
|
||||
private readonly List<Alert> _alerts;
|
||||
private readonly Random _random = new(42);
|
||||
|
||||
public MockAlertDataStore(int alertCount)
|
||||
{
|
||||
_alerts = Enumerable.Range(0, alertCount)
|
||||
.Select(i => new Alert
|
||||
{
|
||||
Id = Guid.NewGuid().ToString(),
|
||||
CveId = $"CVE-2024-{10000 + i}",
|
||||
Severity = _random.Next(0, 4) switch { 0 => "LOW", 1 => "MEDIUM", 2 => "HIGH", _ => "CRITICAL" },
|
||||
Status = "open",
|
||||
CreatedAt = DateTime.UtcNow.AddDays(-_random.Next(1, 30))
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public string GetRandomAlertId() => _alerts[_random.Next(_alerts.Count)].Id;
|
||||
|
||||
public AlertListResult GetAlerts(int page, int pageSize)
|
||||
{
|
||||
// Simulate DB query latency
|
||||
Thread.Sleep(5);
|
||||
|
||||
var skip = (page - 1) * pageSize;
|
||||
return new AlertListResult
|
||||
{
|
||||
Alerts = _alerts.Skip(skip).Take(pageSize).ToList(),
|
||||
TotalCount = _alerts.Count,
|
||||
Page = page,
|
||||
PageSize = pageSize
|
||||
};
|
||||
}
|
||||
|
||||
public Alert GetAlert(string id)
|
||||
{
|
||||
Thread.Sleep(2);
|
||||
return _alerts.First(a => a.Id == id);
|
||||
}
|
||||
|
||||
public DecisionResult RecordDecision(string alertId, DecisionRequest request)
|
||||
{
|
||||
Thread.Sleep(3);
|
||||
return new DecisionResult { Success = true, DecisionId = Guid.NewGuid().ToString() };
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MockEvidenceCache
|
||||
{
|
||||
public EvidenceBundle GetEvidence(string alertId)
|
||||
{
|
||||
// Simulate evidence retrieval latency
|
||||
Thread.Sleep(10);
|
||||
|
||||
return new EvidenceBundle
|
||||
{
|
||||
AlertId = alertId,
|
||||
Reachability = new ReachabilityEvidence
|
||||
{
|
||||
IsReachable = true,
|
||||
Tier = "executed",
|
||||
CallPath = new[] { "main", "process", "vulnerable_func" }
|
||||
},
|
||||
CallStack = new CallStackEvidence
|
||||
{
|
||||
Frames = new[] { "app.dll!Main", "lib.dll!Process", "vulnerable.dll!Sink" }
|
||||
},
|
||||
Provenance = new ProvenanceEvidence
|
||||
{
|
||||
Digest = "sha256:abc123",
|
||||
Registry = "ghcr.io/stellaops"
|
||||
},
|
||||
VexStatus = new VexStatusEvidence
|
||||
{
|
||||
Status = "under_investigation",
|
||||
LastUpdated = DateTime.UtcNow.AddDays(-2)
|
||||
},
|
||||
GraphRevision = new GraphRevisionEvidence
|
||||
{
|
||||
Revision = "graph-v1.2.3",
|
||||
NodeCount = 1500,
|
||||
EdgeCount = 3200
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public static class ReplayTokenGenerator
|
||||
{
|
||||
public static ReplayToken Generate(string alertId, EvidenceBundle evidence)
|
||||
{
|
||||
// Simulate token generation
|
||||
var hash = $"{alertId}:{evidence.Reachability?.Tier}:{evidence.VexStatus?.Status}".GetHashCode();
|
||||
return new ReplayToken
|
||||
{
|
||||
Token = $"replay_{Math.Abs(hash):x8}",
|
||||
AlertId = alertId,
|
||||
GeneratedAt = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Models
|
||||
|
||||
public sealed class Alert
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string CveId { get; set; } = "";
|
||||
public string Severity { get; set; } = "";
|
||||
public string Status { get; set; } = "";
|
||||
public DateTime CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AlertListResult
|
||||
{
|
||||
public List<Alert> Alerts { get; set; } = new();
|
||||
public int TotalCount { get; set; }
|
||||
public int Page { get; set; }
|
||||
public int PageSize { get; set; }
|
||||
}
|
||||
|
||||
public sealed class EvidenceBundle
|
||||
{
|
||||
public string AlertId { get; set; } = "";
|
||||
public ReachabilityEvidence? Reachability { get; set; }
|
||||
public CallStackEvidence? CallStack { get; set; }
|
||||
public ProvenanceEvidence? Provenance { get; set; }
|
||||
public VexStatusEvidence? VexStatus { get; set; }
|
||||
public GraphRevisionEvidence? GraphRevision { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ReachabilityEvidence
|
||||
{
|
||||
public bool IsReachable { get; set; }
|
||||
public string Tier { get; set; } = "";
|
||||
public string[] CallPath { get; set; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
public sealed class CallStackEvidence
|
||||
{
|
||||
public string[] Frames { get; set; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
public sealed class ProvenanceEvidence
|
||||
{
|
||||
public string Digest { get; set; } = "";
|
||||
public string Registry { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class VexStatusEvidence
|
||||
{
|
||||
public string Status { get; set; } = "";
|
||||
public DateTime LastUpdated { get; set; }
|
||||
}
|
||||
|
||||
public sealed class GraphRevisionEvidence
|
||||
{
|
||||
public string Revision { get; set; } = "";
|
||||
public int NodeCount { get; set; }
|
||||
public int EdgeCount { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AlertWithEvidence
|
||||
{
|
||||
public Alert Alert { get; }
|
||||
public EvidenceBundle Evidence { get; }
|
||||
|
||||
public AlertWithEvidence(Alert alert, EvidenceBundle evidence)
|
||||
{
|
||||
Alert = alert;
|
||||
Evidence = evidence;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class DecisionRequest
|
||||
{
|
||||
public string Status { get; set; } = "";
|
||||
public string? Justification { get; set; }
|
||||
public string? ReasonText { get; set; }
|
||||
}
|
||||
|
||||
public sealed class DecisionResult
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public string DecisionId { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class ReplayToken
|
||||
{
|
||||
public string Token { get; set; } = "";
|
||||
public string AlertId { get; set; } = "";
|
||||
public DateTime GeneratedAt { get; set; }
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,431 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3600_0001_0001
|
||||
// Task: TRI-MASTER-0002 - Integration test suite for triage flow
|
||||
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end integration tests for the Triage workflow.
|
||||
/// Tests the complete flow from alert list to decision recording.
|
||||
/// </summary>
|
||||
public sealed class TriageWorkflowIntegrationTests : IClassFixture<ScannerApplicationFactory>
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
public TriageWorkflowIntegrationTests(ScannerApplicationFactory factory)
|
||||
{
|
||||
_client = factory.CreateClient();
|
||||
}
|
||||
|
||||
#region Alert List Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlerts_ReturnsOk_WithPagination()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts?page=1&pageSize=25";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlerts_SupportsBandFilter()
|
||||
{
|
||||
// Arrange - filter by HOT band (high priority)
|
||||
var request = "/api/v1/alerts?band=HOT&page=1&pageSize=25";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlerts_SupportsSeverityFilter()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts?severity=CRITICAL,HIGH&page=1";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlerts_SupportsStatusFilter()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts?status=open&page=1";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlerts_SupportsSortByScore()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts?sortBy=score&sortOrder=desc&page=1";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Alert Detail Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertById_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Evidence Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertEvidence_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345/evidence";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertEvidence_SupportsMinimalFormat()
|
||||
{
|
||||
// Arrange - request minimal evidence bundle
|
||||
var request = "/api/v1/alerts/alert-12345/evidence?format=minimal";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertEvidence_SupportsFullFormat()
|
||||
{
|
||||
// Arrange - request full evidence bundle with graph
|
||||
var request = "/api/v1/alerts/alert-12345/evidence?format=full";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Decision Recording Tests
|
||||
|
||||
[Fact]
|
||||
public async Task RecordDecision_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345/decisions";
|
||||
var decision = new
|
||||
{
|
||||
status = "not_affected",
|
||||
justification = "vulnerable_code_not_in_execute_path",
|
||||
reasonText = "Code path analysis confirms non-reachability"
|
||||
};
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(request, decision);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordDecision_ValidatesStatus()
|
||||
{
|
||||
// Arrange - invalid status
|
||||
var request = "/api/v1/alerts/alert-12345/decisions";
|
||||
var decision = new
|
||||
{
|
||||
status = "invalid_status",
|
||||
justification = "some_justification"
|
||||
};
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(request, decision);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(
|
||||
HttpStatusCode.BadRequest,
|
||||
HttpStatusCode.NotFound,
|
||||
HttpStatusCode.UnprocessableEntity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordDecision_RequiresJustificationForNotAffected()
|
||||
{
|
||||
// Arrange - not_affected without justification
|
||||
var request = "/api/v1/alerts/alert-12345/decisions";
|
||||
var decision = new
|
||||
{
|
||||
status = "not_affected"
|
||||
// Missing justification
|
||||
};
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(request, decision);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(
|
||||
HttpStatusCode.BadRequest,
|
||||
HttpStatusCode.NotFound,
|
||||
HttpStatusCode.UnprocessableEntity);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Audit Trail Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertAudit_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345/audit";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertAudit_SupportsPagination()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-12345/audit?page=1&pageSize=50";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Replay Token Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetReplayToken_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345/replay-token";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task VerifyReplayToken_ReturnsNotFound_WhenTokenInvalid()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/replay/verify";
|
||||
var verifyRequest = new { token = "invalid-token-12345" };
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(request, verifyRequest);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(
|
||||
HttpStatusCode.BadRequest,
|
||||
HttpStatusCode.NotFound,
|
||||
HttpStatusCode.UnprocessableEntity);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Offline Bundle Tests
|
||||
|
||||
[Fact]
|
||||
public async Task DownloadBundle_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345/bundle";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task VerifyBundle_EndpointExists()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/bundles/verify";
|
||||
var bundleData = new { bundleId = "bundle-12345" };
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(request, bundleData);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(
|
||||
HttpStatusCode.OK,
|
||||
HttpStatusCode.BadRequest,
|
||||
HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Diff Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertDiff_ReturnsNotFound_WhenAlertDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/alerts/alert-nonexistent-12345/diff";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAlertDiff_SupportsBaselineParameter()
|
||||
{
|
||||
// Arrange - diff against specific baseline
|
||||
var request = "/api/v1/alerts/alert-12345/diff?baseline=scan-001";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for triage workflow state machine.
|
||||
/// </summary>
|
||||
public sealed class TriageStateMachineTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("open", "not_affected", true)]
|
||||
[InlineData("open", "affected", true)]
|
||||
[InlineData("open", "under_investigation", true)]
|
||||
[InlineData("open", "fixed", true)]
|
||||
[InlineData("not_affected", "open", true)] // Can reopen
|
||||
[InlineData("fixed", "open", true)] // Can reopen
|
||||
[InlineData("affected", "fixed", true)]
|
||||
[InlineData("under_investigation", "not_affected", true)]
|
||||
public void TriageStatus_TransitionIsValid(string from, string to, bool expectedValid)
|
||||
{
|
||||
// Act
|
||||
var isValid = TriageStateMachine.IsValidTransition(from, to);
|
||||
|
||||
// Assert
|
||||
isValid.Should().Be(expectedValid);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not_affected", "vulnerable_code_not_in_execute_path")]
|
||||
[InlineData("not_affected", "vulnerable_code_cannot_be_controlled_by_adversary")]
|
||||
[InlineData("not_affected", "inline_mitigations_already_exist")]
|
||||
public void NotAffectedJustification_MustBeValid(string status, string justification)
|
||||
{
|
||||
// Act
|
||||
var isValid = TriageStateMachine.IsValidJustification(status, justification);
|
||||
|
||||
// Assert
|
||||
isValid.Should().BeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Triage workflow state machine validation.
|
||||
/// </summary>
|
||||
public static class TriageStateMachine
|
||||
{
|
||||
private static readonly HashSet<string> ValidStatuses = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"open",
|
||||
"under_investigation",
|
||||
"affected",
|
||||
"not_affected",
|
||||
"fixed"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> ValidJustifications = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"component_not_present",
|
||||
"vulnerable_code_not_present",
|
||||
"vulnerable_code_not_in_execute_path",
|
||||
"vulnerable_code_cannot_be_controlled_by_adversary",
|
||||
"inline_mitigations_already_exist"
|
||||
};
|
||||
|
||||
public static bool IsValidTransition(string from, string to)
|
||||
{
|
||||
if (!ValidStatuses.Contains(from) || !ValidStatuses.Contains(to))
|
||||
return false;
|
||||
|
||||
// All transitions are valid in this simple model
|
||||
// A more complex implementation might restrict certain paths
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool IsValidJustification(string status, string justification)
|
||||
{
|
||||
if (!string.Equals(status, "not_affected", StringComparison.OrdinalIgnoreCase))
|
||||
return true; // Justification only required for not_affected
|
||||
|
||||
return ValidJustifications.Contains(justification);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
// =============================================================================
|
||||
// ScoreReplayEndpointsTests.cs
|
||||
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
|
||||
// Task: SCORE-REPLAY-013 - Integration tests for score replay endpoint
|
||||
// =============================================================================
|
||||
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for score replay endpoints.
|
||||
/// Per Sprint 3401.0002.0001 - Score Replay & Proof Bundle.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Sprint", "3401.0002")]
|
||||
public sealed class ScoreReplayEndpointsTests : IDisposable
|
||||
{
|
||||
private readonly TestSurfaceSecretsScope _secrets;
|
||||
private readonly ScannerApplicationFactory _factory;
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public ScoreReplayEndpointsTests()
|
||||
{
|
||||
_secrets = new TestSurfaceSecretsScope();
|
||||
_factory = new ScannerApplicationFactory(cfg =>
|
||||
{
|
||||
cfg["scanner:authority:enabled"] = "false";
|
||||
cfg["scanner:scoreReplay:enabled"] = "true";
|
||||
});
|
||||
_client = _factory.CreateClient();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_client.Dispose();
|
||||
_factory.Dispose();
|
||||
_secrets.Dispose();
|
||||
}
|
||||
|
||||
#region POST /score/{scanId}/replay Tests
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/replay returns 404 for unknown scan")]
|
||||
public async Task ReplayScore_UnknownScan_Returns404()
|
||||
{
|
||||
// Arrange
|
||||
var unknownScanId = Guid.NewGuid().ToString();
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsync($"/api/v1/score/{unknownScanId}/replay", null);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/replay returns result for valid scan")]
|
||||
public async Task ReplayScore_ValidScan_ReturnsResult()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
result.Should().NotBeNull();
|
||||
result!.Score.Should().BeInRange(0.0, 1.0);
|
||||
result.RootHash.Should().StartWith("sha256:");
|
||||
result.BundleUri.Should().NotBeNullOrEmpty();
|
||||
result.Deterministic.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/replay is deterministic")]
|
||||
public async Task ReplayScore_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Act - replay twice
|
||||
var response1 = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
var response2 = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
|
||||
// Assert
|
||||
response1.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
response2.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var result1 = await response1.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
var result2 = await response2.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
|
||||
result1!.Score.Should().Be(result2!.Score, "Score should be deterministic");
|
||||
result1.RootHash.Should().Be(result2.RootHash, "RootHash should be deterministic");
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/replay with specific manifest hash")]
|
||||
public async Task ReplayScore_WithManifestHash_UsesSpecificManifest()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Get the manifest hash from the first replay
|
||||
var firstResponse = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
var firstResult = await firstResponse.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
var manifestHash = firstResult!.ManifestHash;
|
||||
|
||||
// Act - replay with specific manifest hash
|
||||
var response = await _client.PostAsJsonAsync(
|
||||
$"/api/v1/score/{scanId}/replay",
|
||||
new { manifestHash });
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
var result = await response.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
result!.ManifestHash.Should().Be(manifestHash);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GET /score/{scanId}/bundle Tests
|
||||
|
||||
[Fact(DisplayName = "GET /score/{scanId}/bundle returns 404 for unknown scan")]
|
||||
public async Task GetBundle_UnknownScan_Returns404()
|
||||
{
|
||||
// Arrange
|
||||
var unknownScanId = Guid.NewGuid().ToString();
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync($"/api/v1/score/{unknownScanId}/bundle");
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /score/{scanId}/bundle returns bundle after replay")]
|
||||
public async Task GetBundle_AfterReplay_ReturnsBundle()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Create a replay first
|
||||
var replayResponse = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
replayResponse.EnsureSuccessStatusCode();
|
||||
var replayResult = await replayResponse.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync($"/api/v1/score/{scanId}/bundle");
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var bundle = await response.Content.ReadFromJsonAsync<ProofBundleResponse>();
|
||||
bundle.Should().NotBeNull();
|
||||
bundle!.RootHash.Should().Be(replayResult!.RootHash);
|
||||
bundle.ManifestDsseValid.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /score/{scanId}/bundle with specific rootHash")]
|
||||
public async Task GetBundle_WithRootHash_ReturnsSpecificBundle()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Create a replay to get a root hash
|
||||
var replayResponse = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
var replayResult = await replayResponse.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
var rootHash = replayResult!.RootHash;
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync($"/api/v1/score/{scanId}/bundle?rootHash={rootHash}");
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
var bundle = await response.Content.ReadFromJsonAsync<ProofBundleResponse>();
|
||||
bundle!.RootHash.Should().Be(rootHash);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region POST /score/{scanId}/verify Tests
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/verify returns valid for correct root hash")]
|
||||
public async Task VerifyBundle_CorrectRootHash_ReturnsValid()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Create a replay
|
||||
var replayResponse = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
var replayResult = await replayResponse.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(
|
||||
$"/api/v1/score/{scanId}/verify",
|
||||
new { expectedRootHash = replayResult!.RootHash });
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
var result = await response.Content.ReadFromJsonAsync<BundleVerifyResponse>();
|
||||
result!.Valid.Should().BeTrue();
|
||||
result.ComputedRootHash.Should().Be(replayResult.RootHash);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/verify returns invalid for wrong root hash")]
|
||||
public async Task VerifyBundle_WrongRootHash_ReturnsInvalid()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Create a replay first
|
||||
await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(
|
||||
$"/api/v1/score/{scanId}/verify",
|
||||
new { expectedRootHash = "sha256:wrong_hash_value" });
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
var result = await response.Content.ReadFromJsonAsync<BundleVerifyResponse>();
|
||||
result!.Valid.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /score/{scanId}/verify validates manifest signature")]
|
||||
public async Task VerifyBundle_ValidatesManifestSignature()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Create a replay
|
||||
var replayResponse = await _client.PostAsync($"/api/v1/score/{scanId}/replay", null);
|
||||
var replayResult = await replayResponse.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
|
||||
// Act
|
||||
var response = await _client.PostAsJsonAsync(
|
||||
$"/api/v1/score/{scanId}/verify",
|
||||
new { expectedRootHash = replayResult!.RootHash });
|
||||
|
||||
// Assert
|
||||
var result = await response.Content.ReadFromJsonAsync<BundleVerifyResponse>();
|
||||
result!.ManifestValid.Should().BeTrue();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Concurrency Tests
|
||||
|
||||
[Fact(DisplayName = "Concurrent replays produce same result")]
|
||||
public async Task ConcurrentReplays_ProduceSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = await CreateTestScanAsync();
|
||||
|
||||
// Act - concurrent replays
|
||||
var tasks = Enumerable.Range(0, 5)
|
||||
.Select(_ => _client.PostAsync($"/api/v1/score/{scanId}/replay", null))
|
||||
.ToList();
|
||||
|
||||
var responses = await Task.WhenAll(tasks);
|
||||
|
||||
// Assert
|
||||
var results = new List<ScoreReplayResponse>();
|
||||
foreach (var response in responses)
|
||||
{
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
var result = await response.Content.ReadFromJsonAsync<ScoreReplayResponse>();
|
||||
results.Add(result!);
|
||||
}
|
||||
|
||||
// All results should have the same score and root hash
|
||||
var firstResult = results[0];
|
||||
foreach (var result in results.Skip(1))
|
||||
{
|
||||
result.Score.Should().Be(firstResult.Score);
|
||||
result.RootHash.Should().Be(firstResult.RootHash);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private async Task<string> CreateTestScanAsync()
|
||||
{
|
||||
var submitResponse = await _client.PostAsJsonAsync("/api/v1/scans", new
|
||||
{
|
||||
image = new { digest = "sha256:test_" + Guid.NewGuid().ToString("N")[..8] }
|
||||
});
|
||||
submitResponse.EnsureSuccessStatusCode();
|
||||
|
||||
var submitPayload = await submitResponse.Content.ReadFromJsonAsync<ScanSubmitResponse>();
|
||||
return submitPayload!.ScanId;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Response Models
|
||||
|
||||
private sealed record ScoreReplayResponse(
|
||||
double Score,
|
||||
string RootHash,
|
||||
string BundleUri,
|
||||
string ManifestHash,
|
||||
DateTimeOffset ReplayedAt,
|
||||
bool Deterministic);
|
||||
|
||||
private sealed record ProofBundleResponse(
|
||||
string ScanId,
|
||||
string RootHash,
|
||||
string BundleUri,
|
||||
bool ManifestDsseValid,
|
||||
DateTimeOffset CreatedAt);
|
||||
|
||||
private sealed record BundleVerifyResponse(
|
||||
bool Valid,
|
||||
string ComputedRootHash,
|
||||
bool ManifestValid,
|
||||
string? ErrorMessage);
|
||||
|
||||
private sealed record ScanSubmitResponse(string ScanId);
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Sprint: SPRINT_3600_0002_0001
|
||||
// Task: UNK-RANK-010 - Integration tests for unknowns API
|
||||
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for the Unknowns API endpoints.
|
||||
/// </summary>
|
||||
public sealed class UnknownsEndpointsTests : IClassFixture<ScannerApplicationFactory>
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
public UnknownsEndpointsTests(ScannerApplicationFactory factory)
|
||||
{
|
||||
_client = factory.CreateClient();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_ReturnsOk_WhenValidRequest()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns?limit=10";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_SupportsPagination()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns?limit=5&offset=0";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_SupportsBandFilter()
|
||||
{
|
||||
// Arrange - filter by HOT band
|
||||
var request = "/api/v1/unknowns?band=HOT&limit=10";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_SupportsSortByScore()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns?sortBy=score&sortOrder=desc&limit=10";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_SupportsSortByLastSeen()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns?sortBy=lastSeen&sortOrder=desc&limit=10";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknownById_ReturnsNotFound_WhenUnknownDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns/unk-nonexistent-12345";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknownEvidence_ReturnsNotFound_WhenUnknownDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns/unk-nonexistent-12345/evidence";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknownHistory_ReturnsNotFound_WhenUnknownDoesNotExist()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns/unk-nonexistent-12345/history";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknownsStats_ReturnsOk()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns/stats";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknownsBandDistribution_ReturnsOk()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns/bands";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_BadRequest_WhenInvalidBand()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns?band=INVALID&limit=10";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.BadRequest, HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUnknowns_BadRequest_WhenLimitTooLarge()
|
||||
{
|
||||
// Arrange
|
||||
var request = "/api/v1/unknowns?limit=10000";
|
||||
|
||||
// Act
|
||||
var response = await _client.GetAsync(request);
|
||||
|
||||
// Assert
|
||||
// Should either reject or cap at max
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.BadRequest, HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for unknowns scoring algorithm.
|
||||
/// </summary>
|
||||
public sealed class UnknownsScoringTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(0.9, 0.8, 0.7, 0.6, 0.5, 0.7)] // High score expected
|
||||
[InlineData(0.1, 0.2, 0.3, 0.2, 0.1, 0.18)] // Low score expected
|
||||
public void ComputeScore_ShouldWeightFactors(
|
||||
double epss, double cvss, double reachability, double freshness, double frequency,
|
||||
double expectedScore)
|
||||
{
|
||||
// Arrange
|
||||
var factors = new UnknownScoringFactors
|
||||
{
|
||||
EpssScore = epss,
|
||||
CvssNormalized = cvss,
|
||||
ReachabilityScore = reachability,
|
||||
FreshnessScore = freshness,
|
||||
FrequencyScore = frequency
|
||||
};
|
||||
|
||||
// Act
|
||||
var score = UnknownsScorer.ComputeScore(factors);
|
||||
|
||||
// Assert
|
||||
score.Should().BeApproximately(expectedScore, 0.1);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0.75, "HOT")]
|
||||
[InlineData(0.50, "WARM")]
|
||||
[InlineData(0.25, "COLD")]
|
||||
public void AssignBand_ShouldMapScoreToBand(double score, string expectedBand)
|
||||
{
|
||||
// Act
|
||||
var band = UnknownsScorer.AssignBand(score);
|
||||
|
||||
// Assert
|
||||
band.Should().Be(expectedBand);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecayScore_ShouldReduceOverTime()
|
||||
{
|
||||
// Arrange
|
||||
var initialScore = 0.8;
|
||||
var daysSinceLastSeen = 7;
|
||||
var decayRate = 0.05; // 5% per day
|
||||
|
||||
// Act
|
||||
var decayedScore = UnknownsScorer.ApplyDecay(initialScore, daysSinceLastSeen, decayRate);
|
||||
|
||||
// Assert
|
||||
decayedScore.Should().BeLessThan(initialScore);
|
||||
decayedScore.Should().BeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scoring factors for unknowns ranking.
|
||||
/// </summary>
|
||||
public record UnknownScoringFactors
|
||||
{
|
||||
public double EpssScore { get; init; }
|
||||
public double CvssNormalized { get; init; }
|
||||
public double ReachabilityScore { get; init; }
|
||||
public double FreshnessScore { get; init; }
|
||||
public double FrequencyScore { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unknowns scoring algorithm.
|
||||
/// </summary>
|
||||
public static class UnknownsScorer
|
||||
{
|
||||
// Weights for 5-factor scoring model
|
||||
private const double EpssWeight = 0.25;
|
||||
private const double CvssWeight = 0.20;
|
||||
private const double ReachabilityWeight = 0.25;
|
||||
private const double FreshnessWeight = 0.15;
|
||||
private const double FrequencyWeight = 0.15;
|
||||
|
||||
public static double ComputeScore(UnknownScoringFactors factors)
|
||||
{
|
||||
return (factors.EpssScore * EpssWeight) +
|
||||
(factors.CvssNormalized * CvssWeight) +
|
||||
(factors.ReachabilityScore * ReachabilityWeight) +
|
||||
(factors.FreshnessScore * FreshnessWeight) +
|
||||
(factors.FrequencyScore * FrequencyWeight);
|
||||
}
|
||||
|
||||
public static string AssignBand(double score)
|
||||
{
|
||||
return score switch
|
||||
{
|
||||
>= 0.7 => "HOT",
|
||||
>= 0.4 => "WARM",
|
||||
_ => "COLD"
|
||||
};
|
||||
}
|
||||
|
||||
public static double ApplyDecay(double score, int daysSinceLastSeen, double decayRate)
|
||||
{
|
||||
var decayFactor = Math.Pow(1 - decayRate, daysSinceLastSeen);
|
||||
return score * decayFactor;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user