Add tests and implement timeline ingestion options with NATS and Redis subscribers
- Introduced `BinaryReachabilityLifterTests` to validate binary lifting functionality. - Created `PackRunWorkerOptions` for configuring worker paths and execution persistence. - Added `TimelineIngestionOptions` for configuring NATS and Redis ingestion transports. - Implemented `NatsTimelineEventSubscriber` for subscribing to NATS events. - Developed `RedisTimelineEventSubscriber` for reading from Redis Streams. - Added `TimelineEnvelopeParser` to normalize incoming event envelopes. - Created unit tests for `TimelineEnvelopeParser` to ensure correct field mapping. - Implemented `TimelineAuthorizationAuditSink` for logging authorization outcomes.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,341 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scanner.Analyzers.Native;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability.Lifters;
|
||||
|
||||
/// <summary>
|
||||
/// Reachability lifter for native binaries (ELF/PE/Mach-O).
|
||||
/// Emits canonical SymbolIDs with file hashes + addresses and anchors code_id attributes
|
||||
/// so richgraph-v1 payloads can be generated even when symbols are stripped.
|
||||
/// </summary>
|
||||
public sealed class BinaryReachabilityLifter : IReachabilityLifter
|
||||
{
|
||||
private static readonly string[] CandidateExtensions = { ".so", ".dll", ".dylib", ".exe", ".bin", ".elf" };
|
||||
|
||||
public string Language => SymbolId.Lang.Binary;
|
||||
|
||||
public async ValueTask LiftAsync(ReachabilityLifterContext context, ReachabilityGraphBuilder builder, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(context.RootPath) || !Directory.Exists(context.RootPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var candidates = EnumerateCandidates(context.RootPath);
|
||||
foreach (var path in candidates)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var info = await AnalyzeBinaryAsync(path, context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
if (info is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
EmitNode(builder, info);
|
||||
EmitDependencies(builder, info);
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> EnumerateCandidates(string rootPath)
|
||||
{
|
||||
return Directory.EnumerateFiles(rootPath, "*", SearchOption.AllDirectories)
|
||||
.Where(IsCandidate)
|
||||
.OrderBy(p => p, StringComparer.Ordinal)
|
||||
.Take(200) // defensive cap for determinism and performance
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsCandidate(string path)
|
||||
{
|
||||
var ext = Path.GetExtension(path);
|
||||
if (!string.IsNullOrWhiteSpace(ext) && CandidateExtensions.Contains(ext, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Span<byte> header = stackalloc byte[4];
|
||||
using var stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
var read = stream.Read(header);
|
||||
if (read < 4)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return IsKnownMagic(header);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsKnownMagic(ReadOnlySpan<byte> header)
|
||||
{
|
||||
return (header[0] == 0x7F && header[1] == (byte)'E' && header[2] == (byte)'L' && header[3] == (byte)'F') // ELF
|
||||
|| (header[0] == (byte)'M' && header[1] == (byte)'Z') // PE
|
||||
|| header.SequenceEqual(stackalloc byte[] { 0xCA, 0xFE, 0xBA, 0xBE }) // Mach-O fat
|
||||
|| header.SequenceEqual(stackalloc byte[] { 0xBE, 0xBA, 0xFE, 0xCA })
|
||||
|| header.SequenceEqual(stackalloc byte[] { 0xFE, 0xED, 0xFA, 0xCF }) // Mach-O 64 LE
|
||||
|| header.SequenceEqual(stackalloc byte[] { 0xCF, 0xFA, 0xED, 0xFE }) // Mach-O 64 BE
|
||||
|| header.SequenceEqual(stackalloc byte[] { 0xFE, 0xED, 0xFA, 0xCE }) // Mach-O 32 LE
|
||||
|| header.SequenceEqual(stackalloc byte[] { 0xCE, 0xFA, 0xED, 0xFE }); // Mach-O 32 BE
|
||||
}
|
||||
|
||||
private static async Task<BinaryInfo?> AnalyzeBinaryAsync(string path, string rootPath, CancellationToken cancellationToken)
|
||||
{
|
||||
byte[] data;
|
||||
try
|
||||
{
|
||||
data = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var shaHex = ComputeSha256Hex(data);
|
||||
var fileHash = $"sha256:{shaHex}";
|
||||
var relativePath = NormalizePath(Path.GetRelativePath(rootPath, path));
|
||||
var format = "binary";
|
||||
NativeBinaryIdentity? identity = null;
|
||||
|
||||
if (TryDetect(data, out var detected))
|
||||
{
|
||||
identity = detected;
|
||||
format = detected.Format switch
|
||||
{
|
||||
NativeFormat.Elf => "elf",
|
||||
NativeFormat.Pe => "pe",
|
||||
NativeFormat.MachO => "macho",
|
||||
_ => "binary"
|
||||
};
|
||||
}
|
||||
|
||||
var dependencies = ParseDependencies(data, format, relativePath, cancellationToken, out var observedBuildId);
|
||||
var buildId = observedBuildId ?? identity?.BuildId ?? identity?.Uuid;
|
||||
|
||||
var symbolId = SymbolId.ForBinaryAddressed(fileHash, ".text", "0x0", Path.GetFileName(path), "static");
|
||||
var codeId = CodeId.ForBinarySegment(format, fileHash, "0x0", data.LongLength, ".text");
|
||||
|
||||
var attributes = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["code_id"] = codeId,
|
||||
["format"] = format,
|
||||
["sha256"] = fileHash
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(buildId))
|
||||
{
|
||||
attributes["build_id"] = buildId!;
|
||||
}
|
||||
|
||||
return new BinaryInfo(
|
||||
SymbolId: symbolId,
|
||||
CodeId: codeId,
|
||||
RelativePath: relativePath,
|
||||
Format: format,
|
||||
FileHash: fileHash,
|
||||
BuildId: buildId,
|
||||
Dependencies: dependencies,
|
||||
DisplayName: Path.GetFileName(path),
|
||||
Attributes: attributes);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<BinaryDependency> ParseDependencies(
|
||||
byte[] data,
|
||||
string format,
|
||||
string evidencePath,
|
||||
CancellationToken cancellationToken,
|
||||
out string? buildId)
|
||||
{
|
||||
buildId = null;
|
||||
var deps = new List<BinaryDependency>();
|
||||
|
||||
switch (format)
|
||||
{
|
||||
case "elf":
|
||||
using (var stream = new MemoryStream(data, writable: false))
|
||||
{
|
||||
if (ElfDynamicSectionParser.TryParse(stream, out var elfInfo, cancellationToken))
|
||||
{
|
||||
buildId = elfInfo.BinaryId;
|
||||
foreach (var dep in elfInfo.Dependencies)
|
||||
{
|
||||
deps.Add(new BinaryDependency(
|
||||
dep.Soname,
|
||||
dep.ReasonCode,
|
||||
EdgeConfidence.Certain,
|
||||
"elf-dynamic",
|
||||
$"file:{evidencePath}:{dep.ReasonCode}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "pe":
|
||||
using (var stream = new MemoryStream(data, writable: false))
|
||||
{
|
||||
if (PeImportParser.TryParse(stream, out var peInfo, cancellationToken))
|
||||
{
|
||||
foreach (var dep in peInfo.Dependencies)
|
||||
{
|
||||
deps.Add(new BinaryDependency(
|
||||
dep.DllName,
|
||||
dep.ReasonCode,
|
||||
EdgeConfidence.Certain,
|
||||
"pe-import",
|
||||
$"file:{evidencePath}:{dep.ReasonCode}"));
|
||||
}
|
||||
|
||||
foreach (var dep in peInfo.DelayLoadDependencies)
|
||||
{
|
||||
deps.Add(new BinaryDependency(
|
||||
dep.DllName,
|
||||
dep.ReasonCode,
|
||||
EdgeConfidence.High,
|
||||
"pe-delayimport",
|
||||
$"file:{evidencePath}:{dep.ReasonCode}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "macho":
|
||||
using (var stream = new MemoryStream(data, writable: false))
|
||||
{
|
||||
if (MachOLoadCommandParser.TryParse(stream, out var machoInfo, cancellationToken))
|
||||
{
|
||||
buildId = machoInfo.Slices.FirstOrDefault(s => !string.IsNullOrWhiteSpace(s.Uuid))?.Uuid;
|
||||
|
||||
foreach (var dep in machoInfo.Slices.SelectMany(s => s.Dependencies))
|
||||
{
|
||||
deps.Add(new BinaryDependency(
|
||||
dep.Path,
|
||||
dep.ReasonCode,
|
||||
EdgeConfidence.Certain,
|
||||
"macho-loadcmd",
|
||||
$"file:{evidencePath}:{dep.ReasonCode}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
var deduped = new Dictionary<string, BinaryDependency>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var dep in deps)
|
||||
{
|
||||
var key = $"{dep.Name}:{dep.Reason}";
|
||||
if (!deduped.TryGetValue(key, out var existing) || existing.Confidence < dep.Confidence)
|
||||
{
|
||||
deduped[key] = dep;
|
||||
}
|
||||
}
|
||||
|
||||
return deduped.Values
|
||||
.OrderBy(d => d.Name, StringComparer.Ordinal)
|
||||
.ThenBy(d => d.Reason, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static void EmitNode(ReachabilityGraphBuilder builder, BinaryInfo info)
|
||||
{
|
||||
builder.AddNode(
|
||||
symbolId: info.SymbolId,
|
||||
lang: SymbolId.Lang.Binary,
|
||||
kind: "binary",
|
||||
display: info.DisplayName,
|
||||
sourceFile: info.RelativePath,
|
||||
attributes: info.Attributes);
|
||||
}
|
||||
|
||||
private static void EmitDependencies(ReachabilityGraphBuilder builder, BinaryInfo info)
|
||||
{
|
||||
if (info.Dependencies.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var dep in info.Dependencies)
|
||||
{
|
||||
var depSymbolId = BuildDependencySymbolId(info.Format, dep.Name);
|
||||
var depAttributes = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["code_id"] = BuildDependencyCodeId(info.Format, dep.Name),
|
||||
["reason"] = dep.Reason
|
||||
};
|
||||
|
||||
builder.AddNode(
|
||||
symbolId: depSymbolId,
|
||||
lang: SymbolId.Lang.Binary,
|
||||
kind: "library",
|
||||
display: dep.Name,
|
||||
attributes: depAttributes);
|
||||
|
||||
builder.AddEdge(
|
||||
from: info.SymbolId,
|
||||
to: depSymbolId,
|
||||
edgeType: EdgeTypes.Import,
|
||||
confidence: dep.Confidence,
|
||||
origin: "static",
|
||||
provenance: dep.Provenance,
|
||||
evidence: dep.Evidence);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryDetect(byte[] data, out NativeBinaryIdentity identity)
|
||||
{
|
||||
using var stream = new MemoryStream(data, writable: false);
|
||||
return NativeFormatDetector.TryDetect(stream, out identity);
|
||||
}
|
||||
|
||||
private static string BuildDependencySymbolId(string format, string name)
|
||||
{
|
||||
var depHash = $"sha256:{ComputeSha256Hex(Encoding.UTF8.GetBytes(name))}";
|
||||
return SymbolId.ForBinaryAddressed(depHash, ".import", "0x0", name, "import");
|
||||
}
|
||||
|
||||
private static string BuildDependencyCodeId(string format, string name)
|
||||
{
|
||||
var depHash = $"sha256:{ComputeSha256Hex(Encoding.UTF8.GetBytes(name))}";
|
||||
return CodeId.ForBinarySegment(format, depHash, "0x0", null, ".import");
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(byte[] data)
|
||||
{
|
||||
var hash = SHA256.HashData(data);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizePath(string path) => path.Replace('\\', '/');
|
||||
|
||||
private sealed record BinaryInfo(
|
||||
string SymbolId,
|
||||
string CodeId,
|
||||
string RelativePath,
|
||||
string Format,
|
||||
string FileHash,
|
||||
string? BuildId,
|
||||
IReadOnlyList<BinaryDependency> Dependencies,
|
||||
string DisplayName,
|
||||
IReadOnlyDictionary<string, string> Attributes);
|
||||
|
||||
private sealed record BinaryDependency(
|
||||
string Name,
|
||||
string Reason,
|
||||
EdgeConfidence Confidence,
|
||||
string Provenance,
|
||||
string Evidence);
|
||||
}
|
||||
@@ -122,6 +122,7 @@ public sealed class ReachabilityLifterRegistry
|
||||
{
|
||||
return new IReachabilityLifter[]
|
||||
{
|
||||
new BinaryReachabilityLifter(),
|
||||
new NodeReachabilityLifter(),
|
||||
new DotNetReachabilityLifter(),
|
||||
// Future lifters:
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Scanner.Cache\StellaOps.Scanner.Cache.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Scanner.Surface.Env\StellaOps.Scanner.Surface.Env.csproj" />
|
||||
<ProjectReference Include="..\..\StellaOps.Scanner.Analyzers.Native\StellaOps.Scanner.Analyzers.Native.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Replay.Core\StellaOps.Replay.Core.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user