feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports
- Introduced a new VEX compact fixture for testing purposes. - Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests. - Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations. - Documented tasks related to the Mirror Creator. - Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs. - Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases. - Added tests for symbol ID normalization in the reachability scanner. - Enhanced console status service with comprehensive unit tests for connection handling and error recovery. - Included Cosign tool version 2.6.0 with checksums for various platforms.
This commit is contained in:
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -37,33 +35,40 @@ public sealed class ReachabilityBuildStageExecutor : IScanStageExecutor
|
||||
}
|
||||
|
||||
var nodeMap = entryTrace.Nodes.ToDictionary(n => n.Id);
|
||||
var symbolMap = new Dictionary<int, string>(nodeMap.Count);
|
||||
|
||||
var unionNodes = new List<ReachabilityUnionNode>(entryTrace.Nodes.Length);
|
||||
foreach (var node in entryTrace.Nodes)
|
||||
{
|
||||
var symbolId = ComputeSymbolId("shell", node.DisplayName, node.Kind.ToString());
|
||||
var command = FormatCommand(node);
|
||||
var symbolId = SymbolId.ForShell(node.DisplayName, command);
|
||||
symbolMap[node.Id] = symbolId;
|
||||
var source = node.Evidence is null
|
||||
? null
|
||||
: new ReachabilitySource("static", "entrytrace", node.Evidence.Path);
|
||||
|
||||
var attributes = new Dictionary<string, string>
|
||||
{
|
||||
["code_id"] = CodeId.FromSymbolId(symbolId)
|
||||
};
|
||||
|
||||
unionNodes.Add(new ReachabilityUnionNode(
|
||||
SymbolId: symbolId,
|
||||
Lang: "shell",
|
||||
Kind: node.Kind.ToString().ToLowerInvariant(),
|
||||
Display: node.DisplayName,
|
||||
Source: source));
|
||||
Source: source,
|
||||
Attributes: attributes));
|
||||
}
|
||||
|
||||
var unionEdges = new List<ReachabilityUnionEdge>(entryTrace.Edges.Length);
|
||||
foreach (var edge in entryTrace.Edges)
|
||||
{
|
||||
if (!nodeMap.TryGetValue(edge.FromNodeId, out var fromNode) || !nodeMap.TryGetValue(edge.ToNodeId, out var toNode))
|
||||
if (!symbolMap.TryGetValue(edge.FromNodeId, out var fromId) || !symbolMap.TryGetValue(edge.ToNodeId, out var toId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var fromId = ComputeSymbolId("shell", fromNode.DisplayName, fromNode.Kind.ToString());
|
||||
var toId = ComputeSymbolId("shell", toNode.DisplayName, toNode.Kind.ToString());
|
||||
unionEdges.Add(new ReachabilityUnionEdge(
|
||||
From: fromId,
|
||||
To: toId,
|
||||
@@ -78,15 +83,13 @@ public sealed class ReachabilityBuildStageExecutor : IScanStageExecutor
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private static string ComputeSymbolId(string lang, string display, string kind)
|
||||
private static string FormatCommand(EntryTraceNode node)
|
||||
{
|
||||
using var sha = SHA256.Create();
|
||||
var input = Encoding.UTF8.GetBytes((display ?? string.Empty) + "|" + (kind ?? string.Empty));
|
||||
var hash = sha.ComputeHash(input);
|
||||
var base64 = Convert.ToBase64String(hash)
|
||||
.TrimEnd('=')
|
||||
.Replace('+', '-')
|
||||
.Replace('/', '_');
|
||||
return $"sym:{lang}:{base64}";
|
||||
if (!node.Arguments.IsDefaultOrEmpty && node.Arguments.Length > 0)
|
||||
{
|
||||
return string.Join(' ', node.Arguments);
|
||||
}
|
||||
|
||||
return node.Kind.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,22 @@ public static class CodeId
|
||||
return Build("dotnet", tuple);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a binary code-id using canonical address + length tuple.
|
||||
/// This aligns with function-level evidence expectations for richgraph-v1.
|
||||
/// </summary>
|
||||
/// <param name="format">Binary format (elf, pe, macho).</param>
|
||||
/// <param name="fileHash">Digest of the binary or object file.</param>
|
||||
/// <param name="address">Virtual address (hex or decimal). Normalized to 0x prefix and lower-case.</param>
|
||||
/// <param name="lengthBytes">Optional length in bytes.</param>
|
||||
/// <param name="section">Optional section name.</param>
|
||||
/// <param name="codeBlockHash">Optional hash of the code block for stripped binaries.</param>
|
||||
public static string ForBinarySegment(string format, string fileHash, string address, long? lengthBytes = null, string? section = null, string? codeBlockHash = null)
|
||||
{
|
||||
var tuple = $"{Norm(format)}\0{Norm(fileHash)}\0{NormalizeAddress(address)}\0{NormalizeLength(lengthBytes)}\0{Norm(section)}\0{Norm(codeBlockHash)}";
|
||||
return Build("binary", tuple);
|
||||
}
|
||||
|
||||
public static string ForNode(string packageName, string entryPath)
|
||||
{
|
||||
var tuple = $"{Norm(packageName)}\0{Norm(entryPath)}";
|
||||
@@ -48,5 +64,48 @@ public static class CodeId
|
||||
return $"code:{lang}:{base64}";
|
||||
}
|
||||
|
||||
private static string NormalizeAddress(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "0x0";
|
||||
}
|
||||
|
||||
var addrText = value.Trim();
|
||||
var isHex = addrText.StartsWith("0x", StringComparison.OrdinalIgnoreCase);
|
||||
if (isHex)
|
||||
{
|
||||
addrText = addrText[2..];
|
||||
}
|
||||
|
||||
if (long.TryParse(addrText, isHex ? System.Globalization.NumberStyles.HexNumber : System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var addrValue))
|
||||
{
|
||||
if (addrValue < 0)
|
||||
{
|
||||
addrValue = 0;
|
||||
}
|
||||
|
||||
return $"0x{addrValue:x}";
|
||||
}
|
||||
|
||||
addrText = addrText.TrimStart('0');
|
||||
if (addrText.Length == 0)
|
||||
{
|
||||
addrText = "0";
|
||||
}
|
||||
|
||||
return $"0x{addrText.ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static string NormalizeLength(long? value)
|
||||
{
|
||||
if (value is null or <= 0)
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
return value.Value.ToString("D", System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static string Norm(string? value) => (value ?? string.Empty).Trim();
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability;
|
||||
|
||||
|
||||
@@ -132,14 +132,21 @@ public static class SymbolId
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a binary symbol ID from ELF/PE/Mach-O components.
|
||||
/// Creates a binary symbol ID from ELF/PE/Mach-O components (legacy overload).
|
||||
/// </summary>
|
||||
/// <param name="buildId">Binary build-id (GNU build-id, PE GUID, Mach-O UUID).</param>
|
||||
/// <param name="section">Section name (e.g., ".text", ".dynsym").</param>
|
||||
/// <param name="symbolName">Symbol name from symbol table.</param>
|
||||
public static string ForBinary(string buildId, string section, string symbolName)
|
||||
=> ForBinaryAddressed(buildId, section, string.Empty, symbolName, "static", null);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a binary symbol ID that includes file hash, section, address, and linkage.
|
||||
/// Aligns with {file:hash, section, addr, name, linkage} tuple used by richgraph-v1.
|
||||
/// </summary>
|
||||
public static string ForBinaryAddressed(string fileHash, string section, string address, string symbolName, string linkage, string? codeBlockHash = null)
|
||||
{
|
||||
var tuple = $"{Norm(buildId)}\0{Norm(section)}\0{Norm(symbolName)}";
|
||||
var tuple = $"{Norm(fileHash)}\0{Norm(section)}\0{NormalizeAddress(address)}\0{Norm(symbolName)}\0{Norm(linkage)}\0{Norm(codeBlockHash)}";
|
||||
return Build(Lang.Binary, tuple);
|
||||
}
|
||||
|
||||
@@ -219,6 +226,40 @@ public static class SymbolId
|
||||
return $"sym:{lang}:{hash}";
|
||||
}
|
||||
|
||||
private static string NormalizeAddress(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "0x0";
|
||||
}
|
||||
|
||||
var addrText = value.Trim();
|
||||
var isHex = addrText.StartsWith("0x", StringComparison.OrdinalIgnoreCase);
|
||||
if (isHex)
|
||||
{
|
||||
addrText = addrText[2..];
|
||||
}
|
||||
|
||||
if (long.TryParse(addrText, isHex ? System.Globalization.NumberStyles.HexNumber : System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var addrValue))
|
||||
{
|
||||
if (addrValue < 0)
|
||||
{
|
||||
addrValue = 0;
|
||||
}
|
||||
|
||||
return $"0x{addrValue:x}";
|
||||
}
|
||||
|
||||
// Fallback to normalized string representation
|
||||
addrText = addrText.TrimStart('0');
|
||||
if (addrText.Length == 0)
|
||||
{
|
||||
addrText = "0";
|
||||
}
|
||||
|
||||
return $"0x{addrText.ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static string ComputeFragment(string tuple)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(tuple);
|
||||
|
||||
@@ -16,8 +16,8 @@ public class RichGraphWriterTests
|
||||
var union = new ReachabilityUnionGraph(
|
||||
Nodes: new[]
|
||||
{
|
||||
new ReachabilityUnionNode("sym:dotnet:B", "dotnet", "method", display: "B"),
|
||||
new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method", display: "A")
|
||||
new ReachabilityUnionNode("sym:dotnet:B", "dotnet", "method", "B"),
|
||||
new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method", "A")
|
||||
},
|
||||
Edges: new[]
|
||||
{
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
using StellaOps.Scanner.Reachability;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability.Tests;
|
||||
|
||||
public class SymbolIdTests
|
||||
{
|
||||
[Fact]
|
||||
public void ForBinaryAddressed_NormalizesAddressAndKeepsLinkage()
|
||||
{
|
||||
var id1 = SymbolId.ForBinaryAddressed("sha256:deadbeef", ".text", "0x0040", "foo", "weak");
|
||||
var id2 = SymbolId.ForBinaryAddressed("sha256:deadbeef", ".text", "0x40", "foo", "weak");
|
||||
|
||||
Assert.Equal(id1, id2);
|
||||
|
||||
var id3 = SymbolId.ForBinaryAddressed("sha256:deadbeef", ".text", "40", "foo", "strong");
|
||||
Assert.NotEqual(id1, id3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CodeIdBinarySegment_NormalizesAddressAndLength()
|
||||
{
|
||||
var cid1 = CodeId.ForBinarySegment("elf", "sha256:abc", "0X0010", 64, ".text");
|
||||
var cid2 = CodeId.ForBinarySegment("elf", "sha256:abc", "16", 64, ".text");
|
||||
|
||||
Assert.Equal(cid1, cid2);
|
||||
|
||||
var cid3 = CodeId.ForBinarySegment("elf", "sha256:abc", "0x20", 32, ".text");
|
||||
Assert.NotEqual(cid1, cid3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SymbolIdForShell_RemainsStableForSameCommand()
|
||||
{
|
||||
var id1 = SymbolId.ForShell("/entrypoint.sh", "python -m app");
|
||||
var id2 = SymbolId.ForShell("/entrypoint.sh", "python -m app");
|
||||
|
||||
Assert.Equal(id1, id2);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user