stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

feat: add security sink detection patterns for JavaScript/TypeScript

Browse Source 
- Introduced `sink-detect.js` with various security sink detection patterns categorized by type (e.g., command injection, SQL injection, file operations).
- Implemented functions to build a lookup map for fast sink detection and to match sink calls against known patterns.
- Added `package-lock.json` for dependency management.
This commit is contained in:
StellaOps Bot
2025-12-22 23:21:21 +02:00
parent 3ba7157b00
commit 5146204f1b
529 changed files with 73579 additions and 5985 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Claims/ClaimsIndex.cs Normal file
View File

@@ -0,0 +1,258 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Benchmark.Claims;
/// <summary>
/// Index of verifiable competitive claims with evidence links.
/// </summary>
public sealed record ClaimsIndex
{
/// <summary>
/// Version of the claims index format.
/// </summary>
public required string Version { get; init; }
/// <summary>
/// When the claims were last verified.
/// </summary>
public required DateTimeOffset LastVerified { get; init; }
/// <summary>
/// The list of claims.
/// </summary>
public required IReadOnlyList<CompetitiveClaim> Claims { get; init; }
/// <summary>
/// Loads a claims index from a JSON file.
/// </summary>
public static async Task<ClaimsIndex?> LoadAsync(string path, CancellationToken ct = default)
{
await using var stream = File.OpenRead(path);
return await JsonSerializer.DeserializeAsync<ClaimsIndex>(stream, JsonOptions, ct);
}
/// <summary>
/// Saves the claims index to a JSON file.
/// </summary>
public async Task SaveAsync(string path, CancellationToken ct = default)
{
await using var stream = File.Create(path);
await JsonSerializer.SerializeAsync(stream, this, JsonOptions, ct);
}
private static readonly JsonSerializerOptions JsonOptions = new()
{
WriteIndented = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
}
/// <summary>
/// A single competitive claim with evidence.
/// </summary>
public sealed record CompetitiveClaim
{
/// <summary>
/// Unique identifier for the claim (e.g., REACH-001).
/// </summary>
public required string ClaimId { get; init; }
/// <summary>
/// Category of the claim.
/// </summary>
public required ClaimCategory Category { get; init; }
/// <summary>
/// The claim statement.
/// </summary>
public required string Claim { get; init; }
/// <summary>
/// Path to evidence file/data.
/// </summary>
public required string EvidencePath { get; init; }
/// <summary>
/// Command to verify the claim.
/// </summary>
public required string VerificationCommand { get; init; }
/// <summary>
/// Status of the claim.
/// </summary>
public required ClaimStatus Status { get; init; }
/// <summary>
/// The specific metric value supporting the claim.
/// </summary>
public string? MetricValue { get; init; }
/// <summary>
/// Comparison baseline (e.g., "vs Trivy 0.50.1").
/// </summary>
public string? Baseline { get; init; }
/// <summary>
/// When the claim was last verified.
/// </summary>
public DateTimeOffset? LastVerified { get; init; }
/// <summary>
/// Notes or caveats about the claim.
/// </summary>
public string? Notes { get; init; }
}
/// <summary>
/// Categories of competitive claims.
/// </summary>
public enum ClaimCategory
{
/// <summary>
/// Reachability analysis claims.
/// </summary>
Reachability,
/// <summary>
/// Precision/accuracy claims.
/// </summary>
Precision,
/// <summary>
/// Recall/coverage claims.
/// </summary>
Recall,
/// <summary>
/// False positive reduction claims.
/// </summary>
FalsePositiveReduction,
/// <summary>
/// Performance/speed claims.
/// </summary>
Performance,
/// <summary>
/// SBOM completeness claims.
/// </summary>
SbomCompleteness,
/// <summary>
/// Explainability claims.
/// </summary>
Explainability,
/// <summary>
/// Reproducibility/determinism claims.
/// </summary>
Reproducibility,
/// <summary>
/// Other claims.
/// </summary>
Other
}
/// <summary>
/// Status of a claim.
/// </summary>
public enum ClaimStatus
{
/// <summary>
/// Claim is verified with current evidence.
/// </summary>
Verified,
/// <summary>
/// Claim needs re-verification.
/// </summary>
NeedsReview,
/// <summary>
/// Claim is pending initial verification.
/// </summary>
Pending,
/// <summary>
/// Claim is outdated and may no longer hold.
/// </summary>
Outdated,
/// <summary>
/// Claim was invalidated by new evidence.
/// </summary>
Invalidated
}
/// <summary>
/// Generates marketing battlecards from benchmark results.
/// </summary>
public sealed class BattlecardGenerator
{
/// <summary>
/// Generates a markdown battlecard from claims and metrics.
/// </summary>
public string Generate(ClaimsIndex claims, IReadOnlyDictionary<string, double> metrics)
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("# Stella Ops Scanner - Competitive Battlecard");
sb.AppendLine();
sb.AppendLine($"*Generated: {DateTimeOffset.UtcNow:yyyy-MM-dd HH:mm:ss} UTC*");
sb.AppendLine();
// Key Differentiators
sb.AppendLine("## Key Differentiators");
sb.AppendLine();
var verifiedClaims = claims.Claims.Where(c => c.Status == ClaimStatus.Verified).ToList();
foreach (var category in Enum.GetValues<ClaimCategory>())
{
var categoryClaims = verifiedClaims.Where(c => c.Category == category).ToList();
if (categoryClaims.Count == 0) continue;
sb.AppendLine($"### {category}");
sb.AppendLine();
foreach (var claim in categoryClaims)
{
sb.AppendLine($"- **{claim.ClaimId}**: {claim.Claim}");
if (claim.MetricValue != null)
sb.AppendLine($" - Metric: {claim.MetricValue}");
if (claim.Baseline != null)
sb.AppendLine($" - Baseline: {claim.Baseline}");
}
sb.AppendLine();
}
// Metrics Summary
sb.AppendLine("## Metrics Summary");
sb.AppendLine();
sb.AppendLine("| Metric | Value |");
sb.AppendLine("|--------|-------|");
foreach (var (name, value) in metrics.OrderBy(kv => kv.Key))
{
sb.AppendLine($"| {name} | {value:P2} |");
}
sb.AppendLine();
// Verification
sb.AppendLine("## Verification");
sb.AppendLine();
sb.AppendLine("All claims can be independently verified using:");
sb.AppendLine();
sb.AppendLine("```bash");
sb.AppendLine("stella bench verify <CLAIM-ID>");
sb.AppendLine("```");
sb.AppendLine();
return sb.ToString();
}
}

129
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Corpus/CorpusManifest.cs Normal file
View File

@@ -0,0 +1,129 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Benchmark.Corpus;
/// <summary>
/// Manifest for the ground-truth corpus of container images.
/// </summary>
public sealed record CorpusManifest
{
/// <summary>
/// Version of the manifest format.
/// </summary>
public required string Version { get; init; }
/// <summary>
/// When the corpus was last updated.
/// </summary>
public required DateTimeOffset LastUpdated { get; init; }
/// <summary>
/// List of images with ground-truth annotations.
/// </summary>
public required IReadOnlyList<ImageGroundTruth> Images { get; init; }
/// <summary>
/// Statistics about the corpus.
/// </summary>
public CorpusStats? Stats { get; init; }
/// <summary>
/// Loads a corpus manifest from a JSON file.
/// </summary>
public static async Task<CorpusManifest?> LoadAsync(string path, CancellationToken ct = default)
{
await using var stream = File.OpenRead(path);
return await JsonSerializer.DeserializeAsync<CorpusManifest>(stream, JsonOptions, ct);
}
/// <summary>
/// Saves the corpus manifest to a JSON file.
/// </summary>
public async Task SaveAsync(string path, CancellationToken ct = default)
{
await using var stream = File.Create(path);
await JsonSerializer.SerializeAsync(stream, this, JsonOptions, ct);
}
private static readonly JsonSerializerOptions JsonOptions = new()
{
WriteIndented = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
}
/// <summary>
/// Ground truth for a single image.
/// </summary>
public sealed record ImageGroundTruth
{
/// <summary>
/// The image digest (sha256:...).
/// </summary>
public required string Digest { get; init; }
/// <summary>
/// Image reference (e.g., alpine:3.18).
/// </summary>
public required string ImageRef { get; init; }
/// <summary>
/// CVEs that are verified true positives (should be reported).
/// </summary>
public required IReadOnlyList<string> TruePositives { get; init; }
/// <summary>
/// CVEs that are verified false positives (should NOT be reported).
/// These are typically backported fixes, unreachable code, etc.
/// </summary>
public required IReadOnlyList<string> FalsePositives { get; init; }
/// <summary>
/// Notes explaining why certain CVEs are classified as FP.
/// Key: CVE ID, Value: Explanation.
/// </summary>
public IReadOnlyDictionary<string, string>? Notes { get; init; }
/// <summary>
/// Image categories (alpine, debian, nodejs, python, etc.).
/// </summary>
public IReadOnlyList<string>? Categories { get; init; }
/// <summary>
/// When the ground truth was last verified.
/// </summary>
public DateTimeOffset? VerifiedAt { get; init; }
/// <summary>
/// Who verified the ground truth.
/// </summary>
public string? VerifiedBy { get; init; }
}
/// <summary>
/// Statistics about the corpus.
/// </summary>
public sealed record CorpusStats
{
/// <summary>
/// Total number of images.
/// </summary>
public required int TotalImages { get; init; }
/// <summary>
/// Breakdown by category.
/// </summary>
public IReadOnlyDictionary<string, int>? ByCategory { get; init; }
/// <summary>
/// Total verified true positives across all images.
/// </summary>
public required int TotalTruePositives { get; init; }
/// <summary>
/// Total verified false positives across all images.
/// </summary>
public required int TotalFalsePositives { get; init; }
}

125
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Corpus/FindingClassification.cs Normal file
View File

@@ -0,0 +1,125 @@
namespace StellaOps.Scanner.Benchmark.Corpus;
/// <summary>
/// Classification of a finding based on ground truth comparison.
/// </summary>
public enum FindingClassification
{
/// <summary>
/// True Positive: Correctly identified vulnerability.
/// </summary>
TruePositive,
/// <summary>
/// False Positive: Incorrectly reported vulnerability.
/// Examples: backported fixes, unreachable code, version mismatch.
/// </summary>
FalsePositive,
/// <summary>
/// True Negative: Correctly not reported (implicit, not commonly tracked).
/// </summary>
TrueNegative,
/// <summary>
/// False Negative: Vulnerability present but not reported by scanner.
/// </summary>
FalseNegative,
/// <summary>
/// Unknown: Not in ground truth, cannot classify.
/// </summary>
Unknown
}
/// <summary>
/// Reasons for false positive classifications.
/// </summary>
public enum FalsePositiveReason
{
/// <summary>
/// The fix was backported by the distribution.
/// </summary>
BackportedFix,
/// <summary>
/// The vulnerable code path is unreachable.
/// </summary>
UnreachableCode,
/// <summary>
/// Version string was incorrectly parsed.
/// </summary>
VersionMismatch,
/// <summary>
/// The vulnerability doesn't apply to this platform.
/// </summary>
PlatformNotAffected,
/// <summary>
/// The vulnerable feature/component is not enabled.
/// </summary>
FeatureDisabled,
/// <summary>
/// Package name collision (different package, same name).
/// </summary>
PackageNameCollision,
/// <summary>
/// Other reason.
/// </summary>
Other
}
/// <summary>
/// Detailed classification report for a finding.
/// </summary>
public sealed record ClassificationReport
{
/// <summary>
/// The CVE ID.
/// </summary>
public required string CveId { get; init; }
/// <summary>
/// The classification.
/// </summary>
public required FindingClassification Classification { get; init; }
/// <summary>
/// For false positives, the reason.
/// </summary>
public FalsePositiveReason? FpReason { get; init; }
/// <summary>
/// Human-readable explanation.
/// </summary>
public string? Explanation { get; init; }
/// <summary>
/// The package name.
/// </summary>
public required string PackageName { get; init; }
/// <summary>
/// The package version.
/// </summary>
public required string PackageVersion { get; init; }
/// <summary>
/// Severity of the vulnerability.
/// </summary>
public required string Severity { get; init; }
/// <summary>
/// The scanner that produced this finding.
/// </summary>
public required string Scanner { get; init; }
/// <summary>
/// The ecosystem (npm, pypi, alpine, etc.).
/// </summary>
public string? Ecosystem { get; init; }
}

125
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Harness/GrypeAdapter.cs Normal file
View File

@@ -0,0 +1,125 @@
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Logging;
namespace StellaOps.Scanner.Benchmark.Harness;
/// <summary>
/// Adapter for Grype vulnerability scanner output.
/// </summary>
public sealed class GrypeAdapter : CompetitorAdapterBase
{
private readonly ILogger<GrypeAdapter> _logger;
private readonly string _grypePath;
public GrypeAdapter(ILogger<GrypeAdapter> logger, string? grypePath = null)
{
_logger = logger;
_grypePath = grypePath ?? "grype";
}
public override string ToolName => "Grype";
public override string ToolVersion => "latest";
public override async Task<IReadOnlyList<NormalizedFinding>> ScanAsync(
string imageRef,
CancellationToken ct = default)
{
_logger.LogInformation("Scanning {Image} with Grype", imageRef);
var startInfo = new ProcessStartInfo
{
FileName = _grypePath,
Arguments = $"--output json {imageRef}",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
using var process = new Process { StartInfo = startInfo };
process.Start();
var output = await process.StandardOutput.ReadToEndAsync(ct);
await process.WaitForExitAsync(ct);
if (process.ExitCode != 0)
{
var error = await process.StandardError.ReadToEndAsync(ct);
_logger.LogError("Grype scan failed: {Error}", error);
return [];
}
return await ParseOutputAsync(output, ct);
}
public override Task<IReadOnlyList<NormalizedFinding>> ParseOutputAsync(
string jsonOutput,
CancellationToken ct = default)
{
var findings = new List<NormalizedFinding>();
try
{
using var doc = JsonDocument.Parse(jsonOutput);
var root = doc.RootElement;
// Grype output structure: { "matches": [ { "vulnerability": {...}, "artifact": {...} } ] }
if (root.TryGetProperty("matches", out var matches))
{
foreach (var match in matches.EnumerateArray())
{
var finding = ParseMatch(match);
if (finding != null)
findings.Add(finding);
}
}
}
catch (JsonException ex)
{
_logger.LogError(ex, "Failed to parse Grype JSON output");
}
return Task.FromResult<IReadOnlyList<NormalizedFinding>>(findings);
}
private NormalizedFinding? ParseMatch(JsonElement match)
{
if (!match.TryGetProperty("vulnerability", out var vuln))
return null;
if (!vuln.TryGetProperty("id", out var idElement))
return null;
var cveId = idElement.GetString();
if (string.IsNullOrEmpty(cveId))
return null;
if (!match.TryGetProperty("artifact", out var artifact))
return null;
var pkgName = artifact.TryGetProperty("name", out var pkg) ? pkg.GetString() : null;
var version = artifact.TryGetProperty("version", out var ver) ? ver.GetString() : null;
var severity = vuln.TryGetProperty("severity", out var sev) ? sev.GetString() : null;
string? fixedVer = null;
if (vuln.TryGetProperty("fix", out var fix) && fix.TryGetProperty("versions", out var fixVersions))
{
var versions = fixVersions.EnumerateArray().Select(v => v.GetString()).ToList();
fixedVer = versions.FirstOrDefault();
}
if (string.IsNullOrEmpty(pkgName) || string.IsNullOrEmpty(version))
return null;
return new NormalizedFinding
{
CveId = cveId,
PackageName = pkgName,
PackageVersion = version,
Severity = NormalizeSeverity(severity),
Source = ToolName,
FixedVersion = fixedVer
};
}
}

67
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Harness/ICompetitorAdapter.cs Normal file
View File

@@ -0,0 +1,67 @@
namespace StellaOps.Scanner.Benchmark.Harness;
/// <summary>
/// Interface for adapting competitor scanner output to normalized findings.
/// </summary>
public interface ICompetitorAdapter
{
/// <summary>
/// The name of the competitor tool.
/// </summary>
string ToolName { get; }
/// <summary>
/// The version of the competitor tool.
/// </summary>
string ToolVersion { get; }
/// <summary>
/// Scans an image and returns normalized findings.
/// </summary>
Task<IReadOnlyList<NormalizedFinding>> ScanAsync(
string imageRef,
CancellationToken ct = default);
/// <summary>
/// Parses existing JSON output from the competitor tool.
/// </summary>
Task<IReadOnlyList<NormalizedFinding>> ParseOutputAsync(
string jsonOutput,
CancellationToken ct = default);
}
/// <summary>
/// Base class for competitor adapters with common functionality.
/// </summary>
public abstract class CompetitorAdapterBase : ICompetitorAdapter
{
public abstract string ToolName { get; }
public abstract string ToolVersion { get; }
public abstract Task<IReadOnlyList<NormalizedFinding>> ScanAsync(
string imageRef,
CancellationToken ct = default);
public abstract Task<IReadOnlyList<NormalizedFinding>> ParseOutputAsync(
string jsonOutput,
CancellationToken ct = default);
/// <summary>
/// Normalizes a severity string to a standard format.
/// </summary>
protected static string NormalizeSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
return "UNKNOWN";
return severity.ToUpperInvariant() switch
{
"CRITICAL" or "CRIT" => "CRITICAL",
"HIGH" or "H" => "HIGH",
"MEDIUM" or "MED" or "M" => "MEDIUM",
"LOW" or "L" => "LOW",
"NEGLIGIBLE" or "NEG" or "INFO" => "NEGLIGIBLE",
_ => "UNKNOWN"
};
}
}

52
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Harness/NormalizedFinding.cs Normal file
View File

@@ -0,0 +1,52 @@
namespace StellaOps.Scanner.Benchmark.Harness;
/// <summary>
/// A normalized finding that can be compared across different scanners.
/// </summary>
public sealed record NormalizedFinding
{
/// <summary>
/// The CVE ID (e.g., CVE-2024-1234).
/// </summary>
public required string CveId { get; init; }
/// <summary>
/// The affected package name.
/// </summary>
public required string PackageName { get; init; }
/// <summary>
/// The installed version of the package.
/// </summary>
public required string PackageVersion { get; init; }
/// <summary>
/// The severity level (CRITICAL, HIGH, MEDIUM, LOW, UNKNOWN).
/// </summary>
public required string Severity { get; init; }
/// <summary>
/// The source scanner that produced this finding.
/// </summary>
public required string Source { get; init; }
/// <summary>
/// The package ecosystem (npm, pypi, maven, etc.).
/// </summary>
public string? Ecosystem { get; init; }
/// <summary>
/// The fixed version if available.
/// </summary>
public string? FixedVersion { get; init; }
/// <summary>
/// Additional metadata from the scanner.
/// </summary>
public IReadOnlyDictionary<string, string>? Metadata { get; init; }
/// <summary>
/// Gets a unique key for this finding for comparison purposes.
/// </summary>
public string UniqueKey => $"{CveId}|{PackageName}|{PackageVersion}".ToLowerInvariant();
}

111
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Harness/SyftAdapter.cs Normal file
View File

@@ -0,0 +1,111 @@
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Logging;
namespace StellaOps.Scanner.Benchmark.Harness;
/// <summary>
/// Adapter for Syft SBOM generator output.
/// Note: Syft generates SBOMs, not vulnerabilities directly.
/// This adapter extracts package information for SBOM comparison.
/// </summary>
public sealed class SyftAdapter : CompetitorAdapterBase
{
private readonly ILogger<SyftAdapter> _logger;
private readonly string _syftPath;
public SyftAdapter(ILogger<SyftAdapter> logger, string? syftPath = null)
{
_logger = logger;
_syftPath = syftPath ?? "syft";
}
public override string ToolName => "Syft";
public override string ToolVersion => "latest";
public override async Task<IReadOnlyList<NormalizedFinding>> ScanAsync(
string imageRef,
CancellationToken ct = default)
{
_logger.LogInformation("Scanning {Image} with Syft", imageRef);
var startInfo = new ProcessStartInfo
{
FileName = _syftPath,
Arguments = $"--output json {imageRef}",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
using var process = new Process { StartInfo = startInfo };
process.Start();
var output = await process.StandardOutput.ReadToEndAsync(ct);
await process.WaitForExitAsync(ct);
if (process.ExitCode != 0)
{
var error = await process.StandardError.ReadToEndAsync(ct);
_logger.LogError("Syft scan failed: {Error}", error);
return [];
}
return await ParseOutputAsync(output, ct);
}
public override Task<IReadOnlyList<NormalizedFinding>> ParseOutputAsync(
string jsonOutput,
CancellationToken ct = default)
{
var findings = new List<NormalizedFinding>();
try
{
using var doc = JsonDocument.Parse(jsonOutput);
var root = doc.RootElement;
// Syft output structure: { "artifacts": [ { "name": "...", "version": "..." } ] }
// Note: Syft doesn't produce vulnerability findings, only SBOM components
// For benchmark purposes, we create placeholder findings for package presence comparison
if (root.TryGetProperty("artifacts", out var artifacts))
{
foreach (var artifact in artifacts.EnumerateArray())
{
var component = ParseArtifact(artifact);
if (component != null)
findings.Add(component);
}
}
}
catch (JsonException ex)
{
_logger.LogError(ex, "Failed to parse Syft JSON output");
}
return Task.FromResult<IReadOnlyList<NormalizedFinding>>(findings);
}
private NormalizedFinding? ParseArtifact(JsonElement artifact)
{
var pkgName = artifact.TryGetProperty("name", out var pkg) ? pkg.GetString() : null;
var version = artifact.TryGetProperty("version", out var ver) ? ver.GetString() : null;
var pkgType = artifact.TryGetProperty("type", out var typeEl) ? typeEl.GetString() : null;
if (string.IsNullOrEmpty(pkgName) || string.IsNullOrEmpty(version))
return null;
// For Syft, we create a pseudo-finding representing package presence
// This is used for SBOM completeness comparison, not vulnerability comparison
return new NormalizedFinding
{
CveId = $"SBOM-COMPONENT-{pkgName}",
PackageName = pkgName,
PackageVersion = version,
Severity = "INFO",
Source = ToolName,
Ecosystem = pkgType
};
}
}

119
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Harness/TrivyAdapter.cs Normal file
View File

@@ -0,0 +1,119 @@
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Logging;
namespace StellaOps.Scanner.Benchmark.Harness;
/// <summary>
/// Adapter for Trivy vulnerability scanner output.
/// </summary>
public sealed class TrivyAdapter : CompetitorAdapterBase
{
private readonly ILogger<TrivyAdapter> _logger;
private readonly string _trivyPath;
public TrivyAdapter(ILogger<TrivyAdapter> logger, string? trivyPath = null)
{
_logger = logger;
_trivyPath = trivyPath ?? "trivy";
}
public override string ToolName => "Trivy";
public override string ToolVersion => "latest";
public override async Task<IReadOnlyList<NormalizedFinding>> ScanAsync(
string imageRef,
CancellationToken ct = default)
{
_logger.LogInformation("Scanning {Image} with Trivy", imageRef);
var startInfo = new ProcessStartInfo
{
FileName = _trivyPath,
Arguments = $"image --format json --quiet {imageRef}",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
using var process = new Process { StartInfo = startInfo };
process.Start();
var output = await process.StandardOutput.ReadToEndAsync(ct);
await process.WaitForExitAsync(ct);
if (process.ExitCode != 0)
{
var error = await process.StandardError.ReadToEndAsync(ct);
_logger.LogError("Trivy scan failed: {Error}", error);
return [];
}
return await ParseOutputAsync(output, ct);
}
public override Task<IReadOnlyList<NormalizedFinding>> ParseOutputAsync(
string jsonOutput,
CancellationToken ct = default)
{
var findings = new List<NormalizedFinding>();
try
{
using var doc = JsonDocument.Parse(jsonOutput);
var root = doc.RootElement;
// Trivy output structure: { "Results": [ { "Vulnerabilities": [...] } ] }
if (root.TryGetProperty("Results", out var results))
{
foreach (var result in results.EnumerateArray())
{
if (!result.TryGetProperty("Vulnerabilities", out var vulnerabilities))
continue;
foreach (var vuln in vulnerabilities.EnumerateArray())
{
var finding = ParseVulnerability(vuln);
if (finding != null)
findings.Add(finding);
}
}
}
}
catch (JsonException ex)
{
_logger.LogError(ex, "Failed to parse Trivy JSON output");
}
return Task.FromResult<IReadOnlyList<NormalizedFinding>>(findings);
}
private NormalizedFinding? ParseVulnerability(JsonElement vuln)
{
if (!vuln.TryGetProperty("VulnerabilityID", out var idElement))
return null;
var cveId = idElement.GetString();
if (string.IsNullOrEmpty(cveId))
return null;
var pkgName = vuln.TryGetProperty("PkgName", out var pkg) ? pkg.GetString() : null;
var version = vuln.TryGetProperty("InstalledVersion", out var ver) ? ver.GetString() : null;
var severity = vuln.TryGetProperty("Severity", out var sev) ? sev.GetString() : null;
var fixedVer = vuln.TryGetProperty("FixedVersion", out var fix) ? fix.GetString() : null;
if (string.IsNullOrEmpty(pkgName) || string.IsNullOrEmpty(version))
return null;
return new NormalizedFinding
{
CveId = cveId,
PackageName = pkgName,
PackageVersion = version,
Severity = NormalizeSeverity(severity),
Source = ToolName,
FixedVersion = fixedVer
};
}
}

152
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Metrics/BenchmarkMetrics.cs Normal file
View File

@@ -0,0 +1,152 @@
namespace StellaOps.Scanner.Benchmark.Metrics;
/// <summary>
/// Benchmark metrics for comparing scanner accuracy.
/// </summary>
public sealed record BenchmarkMetrics
{
/// <summary>
/// Number of true positive findings.
/// </summary>
public required int TruePositives { get; init; }
/// <summary>
/// Number of false positive findings.
/// </summary>
public required int FalsePositives { get; init; }
/// <summary>
/// Number of true negative findings.
/// </summary>
public required int TrueNegatives { get; init; }
/// <summary>
/// Number of false negative findings (missed vulnerabilities).
/// </summary>
public required int FalseNegatives { get; init; }
/// <summary>
/// Precision = TP / (TP + FP).
/// </summary>
public double Precision => TruePositives + FalsePositives > 0
? (double)TruePositives / (TruePositives + FalsePositives)
: 0;
/// <summary>
/// Recall = TP / (TP + FN).
/// </summary>
public double Recall => TruePositives + FalseNegatives > 0
? (double)TruePositives / (TruePositives + FalseNegatives)
: 0;
/// <summary>
/// F1 Score = 2 * (Precision * Recall) / (Precision + Recall).
/// </summary>
public double F1Score => Precision + Recall > 0
? 2 * (Precision * Recall) / (Precision + Recall)
: 0;
/// <summary>
/// Accuracy = (TP + TN) / (TP + TN + FP + FN).
/// </summary>
public double Accuracy
{
get
{
var total = TruePositives + TrueNegatives + FalsePositives + FalseNegatives;
return total > 0 ? (double)(TruePositives + TrueNegatives) / total : 0;
}
}
/// <summary>
/// The scanner tool name.
/// </summary>
public required string ToolName { get; init; }
/// <summary>
/// The image reference that was scanned.
/// </summary>
public string? ImageRef { get; init; }
/// <summary>
/// Timestamp when the benchmark was run.
/// </summary>
public required DateTimeOffset Timestamp { get; init; }
}
/// <summary>
/// Aggregated metrics across multiple images.
/// </summary>
public sealed record AggregatedMetrics
{
/// <summary>
/// The scanner tool name.
/// </summary>
public required string ToolName { get; init; }
/// <summary>
/// Total images scanned.
/// </summary>
public required int TotalImages { get; init; }
/// <summary>
/// Sum of true positives across all images.
/// </summary>
public required int TotalTruePositives { get; init; }
/// <summary>
/// Sum of false positives across all images.
/// </summary>
public required int TotalFalsePositives { get; init; }
/// <summary>
/// Sum of true negatives across all images.
/// </summary>
public required int TotalTrueNegatives { get; init; }
/// <summary>
/// Sum of false negatives across all images.
/// </summary>
public required int TotalFalseNegatives { get; init; }
/// <summary>
/// Aggregate precision.
/// </summary>
public double Precision => TotalTruePositives + TotalFalsePositives > 0
? (double)TotalTruePositives / (TotalTruePositives + TotalFalsePositives)
: 0;
/// <summary>
/// Aggregate recall.
/// </summary>
public double Recall => TotalTruePositives + TotalFalseNegatives > 0
? (double)TotalTruePositives / (TotalTruePositives + TotalFalseNegatives)
: 0;
/// <summary>
/// Aggregate F1 score.
/// </summary>
public double F1Score => Precision + Recall > 0
? 2 * (Precision * Recall) / (Precision + Recall)
: 0;
/// <summary>
/// Breakdown by severity.
/// </summary>
public IReadOnlyDictionary<string, BenchmarkMetrics>? BySeverity { get; init; }
/// <summary>
/// Breakdown by ecosystem.
/// </summary>
public IReadOnlyDictionary<string, BenchmarkMetrics>? ByEcosystem { get; init; }
/// <summary>
/// Individual image metrics.
/// </summary>
public IReadOnlyList<BenchmarkMetrics>? PerImageMetrics { get; init; }
/// <summary>
/// Timestamp when the aggregation was computed.
/// </summary>
public required DateTimeOffset Timestamp { get; init; }
}

153
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/Metrics/MetricsCalculator.cs Normal file
View File

@@ -0,0 +1,153 @@
using StellaOps.Scanner.Benchmark.Corpus;
using StellaOps.Scanner.Benchmark.Harness;
namespace StellaOps.Scanner.Benchmark.Metrics;
/// <summary>
/// Calculates benchmark metrics by comparing scanner findings against ground truth.
/// </summary>
public sealed class MetricsCalculator
{
/// <summary>
/// Calculates metrics for a single image.
/// </summary>
public BenchmarkMetrics Calculate(
string toolName,
string imageRef,
IReadOnlyList<NormalizedFinding> scannerFindings,
ImageGroundTruth groundTruth)
{
var groundTruthPositives = groundTruth.TruePositives
.Select(cve => cve.ToUpperInvariant())
.ToHashSet();
var groundTruthNegatives = groundTruth.FalsePositives
.Select(cve => cve.ToUpperInvariant())
.ToHashSet();
var reportedCves = scannerFindings
.Select(f => f.CveId.ToUpperInvariant())
.ToHashSet();
// True Positives: CVEs correctly identified
var tp = reportedCves.Intersect(groundTruthPositives).Count();
// False Positives: CVEs reported but should not have been
var fp = reportedCves.Intersect(groundTruthNegatives).Count();
// False Negatives: CVEs that should have been reported but weren't
var fn = groundTruthPositives.Except(reportedCves).Count();
// True Negatives: CVEs correctly not reported
var tn = groundTruthNegatives.Except(reportedCves).Count();
return new BenchmarkMetrics
{
ToolName = toolName,
ImageRef = imageRef,
TruePositives = tp,
FalsePositives = fp,
TrueNegatives = tn,
FalseNegatives = fn,
Timestamp = DateTimeOffset.UtcNow
};
}
/// <summary>
/// Aggregates metrics across multiple images.
/// </summary>
public AggregatedMetrics Aggregate(
string toolName,
IReadOnlyList<BenchmarkMetrics> perImageMetrics)
{
var totalTp = perImageMetrics.Sum(m => m.TruePositives);
var totalFp = perImageMetrics.Sum(m => m.FalsePositives);
var totalTn = perImageMetrics.Sum(m => m.TrueNegatives);
var totalFn = perImageMetrics.Sum(m => m.FalseNegatives);
return new AggregatedMetrics
{
ToolName = toolName,
TotalImages = perImageMetrics.Count,
TotalTruePositives = totalTp,
TotalFalsePositives = totalFp,
TotalTrueNegatives = totalTn,
TotalFalseNegatives = totalFn,
PerImageMetrics = perImageMetrics,
Timestamp = DateTimeOffset.UtcNow
};
}
/// <summary>
/// Classifies each finding as TP, FP, TN, or FN.
/// </summary>
public IReadOnlyList<ClassifiedFinding> ClassifyFindings(
IReadOnlyList<NormalizedFinding> scannerFindings,
ImageGroundTruth groundTruth)
{
var groundTruthPositives = groundTruth.TruePositives
.Select(cve => cve.ToUpperInvariant())
.ToHashSet();
var groundTruthNegatives = groundTruth.FalsePositives
.Select(cve => cve.ToUpperInvariant())
.ToHashSet();
var classified = new List<ClassifiedFinding>();
// Classify reported findings
foreach (var finding in scannerFindings)
{
var cveUpper = finding.CveId.ToUpperInvariant();
FindingClassification classification;
string? reason = null;
if (groundTruthPositives.Contains(cveUpper))
{
classification = FindingClassification.TruePositive;
}
else if (groundTruthNegatives.Contains(cveUpper))
{
classification = FindingClassification.FalsePositive;
reason = groundTruth.Notes?.GetValueOrDefault(cveUpper);
}
else
{
// Not in ground truth - treat as unknown
classification = FindingClassification.Unknown;
}
classified.Add(new ClassifiedFinding(finding, classification, reason));
}
// Add false negatives (missed CVEs)
var reportedCves = scannerFindings.Select(f => f.CveId.ToUpperInvariant()).ToHashSet();
foreach (var missedCve in groundTruthPositives.Except(reportedCves))
{
var placeholder = new NormalizedFinding
{
CveId = missedCve,
PackageName = "unknown",
PackageVersion = "unknown",
Severity = "UNKNOWN",
Source = "GroundTruth"
};
classified.Add(new ClassifiedFinding(
placeholder,
FindingClassification.FalseNegative,
"Vulnerability present but not reported by scanner"));
}
return classified;
}
}
/// <summary>
/// A finding with its classification.
/// </summary>
public sealed record ClassifiedFinding(
NormalizedFinding Finding,
FindingClassification Classification,
string? Reason);

13
src/Scanner/__Libraries/StellaOps.Scanner.Benchmark/StellaOps.Scanner.Benchmark.csproj Normal file
View File

@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0" />
<PackageReference Include="System.Text.Json" Version="9.0.0" />
</ItemGroup>
</Project>