Complete batch 012 (golden set diff) and 013 (advisory chat), fix build errors

Sprints completed:
- SPRINT_20260110_012_* (golden set diff layer - 10 sprints)
- SPRINT_20260110_013_* (advisory chat - 4 sprints)

Build fixes applied:
- Fix namespace conflicts with Microsoft.Extensions.Options.Options.Create
- Fix VexDecisionReachabilityIntegrationTests API drift (major rewrite)
- Fix VexSchemaValidationTests FluentAssertions method name
- Fix FixChainGateIntegrationTests ambiguous type references
- Fix AdvisoryAI test files required properties and namespace aliases
- Add stub types for CveMappingController (ICveSymbolMappingService)
- Fix VerdictBuilderService static context issue

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
master
2026-01-11 10:09:07 +02:00
parent a3b2f30a11
commit 7f7eb8b228
232 changed files with 58979 additions and 91 deletions

View File

@@ -0,0 +1,406 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.BinaryIndex.GoldenSet;
/// <summary>
/// Implementation of <see cref="IGoldenSetValidator"/>.
/// </summary>
public sealed partial class GoldenSetValidator : IGoldenSetValidator
{
private readonly ISinkRegistry _sinkRegistry;
private readonly ICveValidator? _cveValidator;
private readonly GoldenSetOptions _options;
private readonly ILogger<GoldenSetValidator> _logger;
/// <summary>
/// Initializes a new instance of <see cref="GoldenSetValidator"/>.
/// </summary>
public GoldenSetValidator(
ISinkRegistry sinkRegistry,
IOptions<GoldenSetOptions> options,
ILogger<GoldenSetValidator> logger,
ICveValidator? cveValidator = null)
{
_sinkRegistry = sinkRegistry ?? throw new ArgumentNullException(nameof(sinkRegistry));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_cveValidator = cveValidator;
}
/// <inheritdoc />
public async Task<GoldenSetValidationResult> ValidateAsync(
GoldenSetDefinition definition,
ValidationOptions? options = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(definition);
options ??= new ValidationOptions
{
ValidateCveExists = _options.Validation.ValidateCveExists,
ValidateSinks = _options.Validation.ValidateSinks,
StrictEdgeFormat = _options.Validation.StrictEdgeFormat,
OfflineMode = _options.Validation.OfflineMode
};
var errors = new List<ValidationError>();
var warnings = new List<ValidationWarning>();
// 1. Required fields validation
ValidateRequiredFields(definition, errors);
// 2. ID format validation
ValidateIdFormat(definition.Id, errors);
// 3. CVE existence validation (if enabled and online)
if (options.ValidateCveExists && !options.OfflineMode && _cveValidator is not null)
{
await ValidateCveExistsAsync(definition.Id, errors, ct);
}
// 4. Targets validation
ValidateTargets(definition.Targets, options, errors, warnings);
// 5. Metadata validation
ValidateMetadata(definition.Metadata, errors, warnings);
// If there are errors, return failure
if (errors.Count > 0)
{
_logger.LogDebug("Golden set {Id} validation failed with {ErrorCount} errors", definition.Id, errors.Count);
return GoldenSetValidationResult.Failure(
errors.ToImmutableArray(),
warnings.ToImmutableArray());
}
// Compute content digest
var digest = ComputeContentDigest(definition);
_logger.LogDebug("Golden set {Id} validated successfully with digest {Digest}", definition.Id, digest);
return GoldenSetValidationResult.Success(
definition,
digest,
warnings.ToImmutableArray());
}
/// <inheritdoc />
public async Task<GoldenSetValidationResult> ValidateYamlAsync(
string yamlContent,
ValidationOptions? options = null,
CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(yamlContent);
try
{
var definition = GoldenSetYamlSerializer.Deserialize(yamlContent);
return await ValidateAsync(definition, options, ct);
}
catch (Exception ex) when (ex is YamlDotNet.Core.YamlException or InvalidOperationException)
{
_logger.LogDebug(ex, "YAML parsing failed");
return GoldenSetValidationResult.Failure(
[new ValidationError(ValidationErrorCodes.YamlParseError, ex.Message)]);
}
}
private static void ValidateRequiredFields(GoldenSetDefinition definition, List<ValidationError> errors)
{
if (string.IsNullOrWhiteSpace(definition.Id))
{
errors.Add(new ValidationError(ValidationErrorCodes.RequiredFieldMissing, "Id is required", "id"));
}
if (string.IsNullOrWhiteSpace(definition.Component))
{
errors.Add(new ValidationError(ValidationErrorCodes.RequiredFieldMissing, "Component is required", "component"));
}
if (definition.Targets.IsDefault || definition.Targets.Length == 0)
{
errors.Add(new ValidationError(ValidationErrorCodes.NoTargets, "At least one target is required", "targets"));
}
if (definition.Metadata is null)
{
errors.Add(new ValidationError(ValidationErrorCodes.RequiredFieldMissing, "Metadata is required", "metadata"));
}
}
private static void ValidateIdFormat(string? id, List<ValidationError> errors)
{
if (string.IsNullOrWhiteSpace(id))
{
return; // Already reported as missing
}
// Accept CVE-YYYY-NNNN or GHSA-xxxx-xxxx-xxxx formats
if (!CveIdRegex().IsMatch(id) && !GhsaIdRegex().IsMatch(id))
{
errors.Add(new ValidationError(
ValidationErrorCodes.InvalidIdFormat,
string.Format(CultureInfo.InvariantCulture, "Invalid ID format: {0}. Expected CVE-YYYY-NNNN or GHSA-xxxx-xxxx-xxxx.", id),
"id"));
}
}
private async Task ValidateCveExistsAsync(string id, List<ValidationError> errors, CancellationToken ct)
{
if (_cveValidator is null)
{
return;
}
try
{
var exists = await _cveValidator.ExistsAsync(id, ct);
if (!exists)
{
errors.Add(new ValidationError(
ValidationErrorCodes.CveNotFound,
string.Format(CultureInfo.InvariantCulture, "CVE {0} not found in NVD/OSV", id),
"id"));
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to validate CVE existence for {Id}", id);
// Don't add error - network failures shouldn't block validation
}
}
private void ValidateTargets(
ImmutableArray<VulnerableTarget> targets,
ValidationOptions options,
List<ValidationError> errors,
List<ValidationWarning> warnings)
{
if (targets.IsDefault)
{
return;
}
for (int i = 0; i < targets.Length; i++)
{
var target = targets[i];
var path = string.Format(CultureInfo.InvariantCulture, "targets[{0}]", i);
// Function name required
if (string.IsNullOrWhiteSpace(target.FunctionName))
{
errors.Add(new ValidationError(
ValidationErrorCodes.EmptyFunctionName,
"Function name is required",
string.Concat(path, ".function")));
}
// Edge format validation
if (options.StrictEdgeFormat && !target.Edges.IsDefault)
{
foreach (var edge in target.Edges)
{
if (!IsValidEdgeFormat(edge))
{
errors.Add(new ValidationError(
ValidationErrorCodes.InvalidEdgeFormat,
string.Format(CultureInfo.InvariantCulture, "Invalid edge format: {0}. Expected 'bbN->bbM'.", edge),
string.Concat(path, ".edges")));
}
}
}
// Sink validation
if (options.ValidateSinks && !target.Sinks.IsDefault)
{
foreach (var sink in target.Sinks)
{
if (!_sinkRegistry.IsKnownSink(sink))
{
warnings.Add(new ValidationWarning(
ValidationWarningCodes.UnknownSink,
string.Format(CultureInfo.InvariantCulture, "Sink '{0}' not in registry", sink),
string.Concat(path, ".sinks")));
}
}
}
// Constant format validation
if (!target.Constants.IsDefault)
{
foreach (var constant in target.Constants)
{
if (!IsValidConstant(constant))
{
warnings.Add(new ValidationWarning(
ValidationWarningCodes.MalformedConstant,
string.Format(CultureInfo.InvariantCulture, "Constant '{0}' may be malformed", constant),
string.Concat(path, ".constants")));
}
}
}
// Warn if no edges or sinks
if (target.Edges.IsDefaultOrEmpty)
{
warnings.Add(new ValidationWarning(
ValidationWarningCodes.NoEdges,
"No edges defined for target",
path));
}
if (target.Sinks.IsDefaultOrEmpty)
{
warnings.Add(new ValidationWarning(
ValidationWarningCodes.NoSinks,
"No sinks defined for target",
path));
}
}
}
private static void ValidateMetadata(
GoldenSetMetadata? metadata,
List<ValidationError> errors,
List<ValidationWarning> warnings)
{
if (metadata is null)
{
return; // Already reported as missing
}
if (string.IsNullOrWhiteSpace(metadata.AuthorId))
{
errors.Add(new ValidationError(
ValidationErrorCodes.RequiredFieldMissing,
"Author ID is required",
"metadata.author_id"));
}
if (string.IsNullOrWhiteSpace(metadata.SourceRef))
{
errors.Add(new ValidationError(
ValidationErrorCodes.RequiredFieldMissing,
"Source reference is required",
"metadata.source_ref"));
}
// Validate timestamps are not default/min values
if (metadata.CreatedAt == default || metadata.CreatedAt == DateTimeOffset.MinValue)
{
errors.Add(new ValidationError(
ValidationErrorCodes.InvalidTimestamp,
"Created timestamp is required and must be valid",
"metadata.created_at"));
}
// Validate schema version format
if (!string.IsNullOrEmpty(metadata.SchemaVersion) && !SchemaVersionRegex().IsMatch(metadata.SchemaVersion))
{
errors.Add(new ValidationError(
ValidationErrorCodes.InvalidSchemaVersion,
string.Format(CultureInfo.InvariantCulture, "Invalid schema version format: {0}", metadata.SchemaVersion),
"metadata.schema_version"));
}
// Warn if source ref doesn't look like a URL
if (!string.IsNullOrWhiteSpace(metadata.SourceRef) &&
!Uri.TryCreate(metadata.SourceRef, UriKind.Absolute, out _) &&
!metadata.SourceRef.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
warnings.Add(new ValidationWarning(
ValidationWarningCodes.InvalidSourceRef,
"Source reference may be invalid (not a URL or hash)",
"metadata.source_ref"));
}
}
private static bool IsValidEdgeFormat(BasicBlockEdge edge)
{
// Accept bb-prefixed blocks or generic block identifiers
return edge.From.StartsWith("bb", StringComparison.Ordinal) &&
edge.To.StartsWith("bb", StringComparison.Ordinal);
}
private static bool IsValidConstant(string constant)
{
if (string.IsNullOrWhiteSpace(constant))
{
return false;
}
// Accept hex (0x...), decimal, or quoted string literals
if (constant.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
{
// Hex constant - verify valid hex digits after prefix
return constant.Length > 2 && constant[2..].All(char.IsAsciiHexDigit);
}
// Accept decimal numbers or any non-empty string
return !string.IsNullOrWhiteSpace(constant);
}
private static string ComputeContentDigest(GoldenSetDefinition definition)
{
// Create a canonical representation for hashing
// We exclude ContentDigest from the hash computation
var canonical = new
{
id = definition.Id,
component = definition.Component,
targets = definition.Targets.Select(t => new
{
function = t.FunctionName,
edges = t.Edges.Select(e => e.ToString()).OrderBy(e => e, StringComparer.Ordinal).ToArray(),
sinks = t.Sinks.OrderBy(s => s, StringComparer.Ordinal).ToArray(),
constants = t.Constants.OrderBy(c => c, StringComparer.Ordinal).ToArray(),
taint_invariant = t.TaintInvariant,
source_file = t.SourceFile,
source_line = t.SourceLine
}).OrderBy(t => t.function, StringComparer.Ordinal).ToArray(),
witness = definition.Witness is null ? null : new
{
arguments = definition.Witness.Arguments.ToArray(),
invariant = definition.Witness.Invariant,
poc_file_ref = definition.Witness.PocFileRef
},
metadata = new
{
author_id = definition.Metadata.AuthorId,
created_at = definition.Metadata.CreatedAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture),
source_ref = definition.Metadata.SourceRef,
reviewed_by = definition.Metadata.ReviewedBy,
reviewed_at = definition.Metadata.ReviewedAt?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture),
tags = definition.Metadata.Tags.OrderBy(t => t, StringComparer.Ordinal).ToArray(),
schema_version = definition.Metadata.SchemaVersion
}
};
var json = JsonSerializer.Serialize(canonical, CanonicalJsonOptions);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(json));
return string.Concat("sha256:", Convert.ToHexStringLower(hash));
}
private static readonly JsonSerializerOptions CanonicalJsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
WriteIndented = false,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
};
[GeneratedRegex(GoldenSetConstants.CveIdPattern)]
private static partial Regex CveIdRegex();
[GeneratedRegex(GoldenSetConstants.GhsaIdPattern)]
private static partial Regex GhsaIdRegex();
[GeneratedRegex(@"^\d+\.\d+\.\d+$")]
private static partial Regex SchemaVersionRegex();
}

View File

@@ -0,0 +1,64 @@
namespace StellaOps.BinaryIndex.GoldenSet;
/// <summary>
/// Service for validating CVE existence in external databases.
/// </summary>
public interface ICveValidator
{
/// <summary>
/// Checks if a vulnerability ID exists in NVD/OSV/GHSA.
/// </summary>
/// <param name="vulnerabilityId">The vulnerability ID to check.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>True if the vulnerability exists; otherwise, false.</returns>
Task<bool> ExistsAsync(string vulnerabilityId, CancellationToken ct = default);
/// <summary>
/// Gets vulnerability details if available.
/// </summary>
/// <param name="vulnerabilityId">The vulnerability ID to look up.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>Vulnerability details or null if not found.</returns>
Task<CveDetails?> GetDetailsAsync(string vulnerabilityId, CancellationToken ct = default);
}
/// <summary>
/// Basic CVE details from external sources.
/// </summary>
public sealed record CveDetails
{
/// <summary>
/// Vulnerability ID.
/// </summary>
public required string Id { get; init; }
/// <summary>
/// Description of the vulnerability.
/// </summary>
public string? Description { get; init; }
/// <summary>
/// Published date.
/// </summary>
public DateTimeOffset? PublishedDate { get; init; }
/// <summary>
/// Last modified date.
/// </summary>
public DateTimeOffset? ModifiedDate { get; init; }
/// <summary>
/// Associated CWE IDs.
/// </summary>
public IReadOnlyList<string> CweIds { get; init; } = [];
/// <summary>
/// CVSS score if available.
/// </summary>
public double? CvssScore { get; init; }
/// <summary>
/// Source of the data (nvd, osv, ghsa).
/// </summary>
public required string Source { get; init; }
}

View File

@@ -0,0 +1,198 @@
using System.Collections.Immutable;
namespace StellaOps.BinaryIndex.GoldenSet;
/// <summary>
/// Service for validating golden set definitions.
/// </summary>
public interface IGoldenSetValidator
{
/// <summary>
/// Validates a golden set definition.
/// </summary>
/// <param name="definition">The definition to validate.</param>
/// <param name="options">Validation options.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>Validation result with errors and warnings.</returns>
Task<GoldenSetValidationResult> ValidateAsync(
GoldenSetDefinition definition,
ValidationOptions? options = null,
CancellationToken ct = default);
/// <summary>
/// Validates a golden set from YAML content.
/// </summary>
/// <param name="yamlContent">YAML string to parse and validate.</param>
/// <param name="options">Validation options.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>Validation result with errors and warnings.</returns>
Task<GoldenSetValidationResult> ValidateYamlAsync(
string yamlContent,
ValidationOptions? options = null,
CancellationToken ct = default);
}
/// <summary>
/// Result of golden set validation.
/// </summary>
public sealed record GoldenSetValidationResult
{
/// <summary>
/// Whether the definition is valid (no errors).
/// </summary>
public required bool IsValid { get; init; }
/// <summary>
/// Validation errors (must be empty for IsValid to be true).
/// </summary>
public ImmutableArray<ValidationError> Errors { get; init; } = [];
/// <summary>
/// Validation warnings (do not affect IsValid).
/// </summary>
public ImmutableArray<ValidationWarning> Warnings { get; init; } = [];
/// <summary>
/// Parsed definition with computed content digest (null if errors).
/// </summary>
public GoldenSetDefinition? ParsedDefinition { get; init; }
/// <summary>
/// Content digest of the validated definition (null if errors).
/// </summary>
public string? ContentDigest { get; init; }
/// <summary>
/// Creates a successful validation result.
/// </summary>
public static GoldenSetValidationResult Success(
GoldenSetDefinition definition,
string contentDigest,
ImmutableArray<ValidationWarning> warnings = default) => new()
{
IsValid = true,
ParsedDefinition = definition with { ContentDigest = contentDigest },
ContentDigest = contentDigest,
Warnings = warnings.IsDefault ? [] : warnings
};
/// <summary>
/// Creates a failed validation result.
/// </summary>
public static GoldenSetValidationResult Failure(
ImmutableArray<ValidationError> errors,
ImmutableArray<ValidationWarning> warnings = default) => new()
{
IsValid = false,
Errors = errors,
Warnings = warnings.IsDefault ? [] : warnings
};
}
/// <summary>
/// A validation error (blocks acceptance).
/// </summary>
/// <param name="Code">Error code for programmatic handling.</param>
/// <param name="Message">Human-readable error message.</param>
/// <param name="Path">JSON path to the problematic field.</param>
public sealed record ValidationError(
string Code,
string Message,
string? Path = null);
/// <summary>
/// A validation warning (informational, does not block).
/// </summary>
/// <param name="Code">Warning code for programmatic handling.</param>
/// <param name="Message">Human-readable warning message.</param>
/// <param name="Path">JSON path to the problematic field.</param>
public sealed record ValidationWarning(
string Code,
string Message,
string? Path = null);
/// <summary>
/// Options controlling validation behavior.
/// </summary>
public sealed record ValidationOptions
{
/// <summary>
/// Validate that the CVE exists in NVD/OSV (requires network).
/// </summary>
public bool ValidateCveExists { get; init; } = true;
/// <summary>
/// Validate that sinks are in the registry.
/// </summary>
public bool ValidateSinks { get; init; } = true;
/// <summary>
/// Validate edge format strictly (must match bbN->bbM).
/// </summary>
public bool StrictEdgeFormat { get; init; } = true;
/// <summary>
/// Skip network calls (air-gap mode).
/// </summary>
public bool OfflineMode { get; init; } = false;
}
/// <summary>
/// Well-known validation error codes.
/// </summary>
public static class ValidationErrorCodes
{
/// <summary>Required field is missing.</summary>
public const string RequiredFieldMissing = "REQUIRED_FIELD_MISSING";
/// <summary>CVE not found in external databases.</summary>
public const string CveNotFound = "CVE_NOT_FOUND";
/// <summary>Invalid vulnerability ID format.</summary>
public const string InvalidIdFormat = "INVALID_ID_FORMAT";
/// <summary>Empty or whitespace function name.</summary>
public const string EmptyFunctionName = "EMPTY_FUNCTION_NAME";
/// <summary>Invalid basic block edge format.</summary>
public const string InvalidEdgeFormat = "INVALID_EDGE_FORMAT";
/// <summary>No targets defined.</summary>
public const string NoTargets = "NO_TARGETS";
/// <summary>Invalid constant format.</summary>
public const string InvalidConstant = "INVALID_CONSTANT";
/// <summary>Invalid timestamp format.</summary>
public const string InvalidTimestamp = "INVALID_TIMESTAMP";
/// <summary>Invalid schema version.</summary>
public const string InvalidSchemaVersion = "INVALID_SCHEMA_VERSION";
/// <summary>YAML parsing failed.</summary>
public const string YamlParseError = "YAML_PARSE_ERROR";
}
/// <summary>
/// Well-known validation warning codes.
/// </summary>
public static class ValidationWarningCodes
{
/// <summary>Sink not found in registry.</summary>
public const string UnknownSink = "UNKNOWN_SINK";
/// <summary>Edge format may be non-standard.</summary>
public const string NonStandardEdge = "NON_STANDARD_EDGE";
/// <summary>Constant may be malformed.</summary>
public const string MalformedConstant = "MALFORMED_CONSTANT";
/// <summary>Source reference may be invalid.</summary>
public const string InvalidSourceRef = "INVALID_SOURCE_REF";
/// <summary>No sinks defined for target.</summary>
public const string NoSinks = "NO_SINKS";
/// <summary>No edges defined for target.</summary>
public const string NoEdges = "NO_EDGES";
}