Add property-based tests for SBOM/VEX document ordering and Unicode normalization determinism
- Implement `SbomVexOrderingDeterminismProperties` for testing component list and vulnerability metadata hash consistency. - Create `UnicodeNormalizationDeterminismProperties` to validate NFC normalization and Unicode string handling. - Add project file for `StellaOps.Testing.Determinism.Properties` with necessary dependencies. - Introduce CI/CD template validation tests including YAML syntax checks and documentation content verification. - Create validation script for CI/CD templates ensuring all required files and structures are present.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,224 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.BinaryIndex.FixIndex.Models;
|
||||
|
||||
namespace StellaOps.BinaryIndex.FixIndex.Parsers;
|
||||
|
||||
/// <summary>
|
||||
/// Parses RPM spec file changelog sections for CVE mentions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// RPM changelog format:
|
||||
/// %changelog
|
||||
/// * Mon Jan 01 2024 Packager <email> - 1.2.3-4
|
||||
/// - Fix CVE-2024-1234
|
||||
/// </remarks>
|
||||
public sealed partial class RpmChangelogParser : IChangelogParser
|
||||
{
|
||||
[GeneratedRegex(@"\bCVE-\d{4}-\d{4,7}\b", RegexOptions.Compiled)]
|
||||
private static partial Regex CvePatternRegex();
|
||||
|
||||
[GeneratedRegex(@"^\*\s+\w{3}\s+\w{3}\s+\d{1,2}\s+\d{4}\s+(.+?)\s+-\s+(\S+)", RegexOptions.Compiled)]
|
||||
private static partial Regex EntryHeaderPatternRegex();
|
||||
|
||||
[GeneratedRegex(@"^%changelog\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex ChangelogStartPatternRegex();
|
||||
|
||||
[GeneratedRegex(@"^%\w+", RegexOptions.Compiled)]
|
||||
private static partial Regex SectionStartPatternRegex();
|
||||
|
||||
/// <summary>
|
||||
/// Parses the top entry of an RPM spec changelog for CVE mentions.
|
||||
/// </summary>
|
||||
public IEnumerable<FixEvidence> ParseTopEntry(
|
||||
string specContent,
|
||||
string distro,
|
||||
string release,
|
||||
string sourcePkg)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(specContent))
|
||||
yield break;
|
||||
|
||||
var lines = specContent.Split('\n');
|
||||
var inChangelog = false;
|
||||
var inFirstEntry = false;
|
||||
string? currentVersion = null;
|
||||
var entryLines = new List<string>();
|
||||
|
||||
foreach (var line in lines)
|
||||
{
|
||||
// Detect %changelog start
|
||||
if (ChangelogStartPatternRegex().IsMatch(line))
|
||||
{
|
||||
inChangelog = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inChangelog)
|
||||
continue;
|
||||
|
||||
// Exit on new section (e.g., %files, %prep)
|
||||
if (SectionStartPatternRegex().IsMatch(line) && !ChangelogStartPatternRegex().IsMatch(line))
|
||||
break;
|
||||
|
||||
// Detect entry header: * Day Mon DD YYYY Author <email> - version
|
||||
var headerMatch = EntryHeaderPatternRegex().Match(line);
|
||||
if (headerMatch.Success)
|
||||
{
|
||||
if (inFirstEntry)
|
||||
{
|
||||
// We've hit the second entry, stop processing
|
||||
break;
|
||||
}
|
||||
|
||||
inFirstEntry = true;
|
||||
currentVersion = headerMatch.Groups[2].Value;
|
||||
entryLines.Add(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inFirstEntry)
|
||||
{
|
||||
entryLines.Add(line);
|
||||
}
|
||||
}
|
||||
|
||||
if (currentVersion == null || entryLines.Count == 0)
|
||||
yield break;
|
||||
|
||||
var entryText = string.Join('\n', entryLines);
|
||||
var cves = CvePatternRegex().Matches(entryText)
|
||||
.Select(m => m.Value)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
foreach (var cve in cves)
|
||||
{
|
||||
yield return new FixEvidence
|
||||
{
|
||||
Distro = distro,
|
||||
Release = release,
|
||||
SourcePkg = sourcePkg,
|
||||
CveId = cve,
|
||||
State = FixState.Fixed,
|
||||
FixedVersion = currentVersion,
|
||||
Method = FixMethod.Changelog,
|
||||
Confidence = 0.75m, // RPM changelogs are less structured than Debian
|
||||
Evidence = new ChangelogEvidence
|
||||
{
|
||||
File = "*.spec",
|
||||
Version = currentVersion,
|
||||
Excerpt = entryText.Length > 2000 ? entryText[..2000] : entryText,
|
||||
LineNumber = null
|
||||
},
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses the full RPM spec changelog for all CVE mentions with their versions.
|
||||
/// </summary>
|
||||
public IEnumerable<FixEvidence> ParseAllEntries(
|
||||
string specContent,
|
||||
string distro,
|
||||
string release,
|
||||
string sourcePkg)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(specContent))
|
||||
yield break;
|
||||
|
||||
var lines = specContent.Split('\n');
|
||||
var inChangelog = false;
|
||||
string? currentVersion = null;
|
||||
var currentEntry = new List<string>();
|
||||
|
||||
foreach (var line in lines)
|
||||
{
|
||||
// Detect %changelog start
|
||||
if (ChangelogStartPatternRegex().IsMatch(line))
|
||||
{
|
||||
inChangelog = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inChangelog)
|
||||
continue;
|
||||
|
||||
// Exit on new section
|
||||
if (SectionStartPatternRegex().IsMatch(line) && !ChangelogStartPatternRegex().IsMatch(line))
|
||||
{
|
||||
// Process last entry
|
||||
if (currentVersion != null && currentEntry.Count > 0)
|
||||
{
|
||||
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, distro, release, sourcePkg))
|
||||
yield return fix;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Detect entry header
|
||||
var headerMatch = EntryHeaderPatternRegex().Match(line);
|
||||
if (headerMatch.Success)
|
||||
{
|
||||
// Process previous entry
|
||||
if (currentVersion != null && currentEntry.Count > 0)
|
||||
{
|
||||
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, distro, release, sourcePkg))
|
||||
yield return fix;
|
||||
}
|
||||
|
||||
currentVersion = headerMatch.Groups[2].Value;
|
||||
currentEntry = [line];
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentVersion != null)
|
||||
{
|
||||
currentEntry.Add(line);
|
||||
}
|
||||
}
|
||||
|
||||
// Process final entry if exists
|
||||
if (currentVersion != null && currentEntry.Count > 0)
|
||||
{
|
||||
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, distro, release, sourcePkg))
|
||||
yield return fix;
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<FixEvidence> ExtractCvesFromEntry(
|
||||
List<string> entryLines,
|
||||
string version,
|
||||
string distro,
|
||||
string release,
|
||||
string sourcePkg)
|
||||
{
|
||||
var entryText = string.Join('\n', entryLines);
|
||||
var cves = CvePatternRegex().Matches(entryText)
|
||||
.Select(m => m.Value)
|
||||
.Distinct();
|
||||
|
||||
foreach (var cve in cves)
|
||||
{
|
||||
yield return new FixEvidence
|
||||
{
|
||||
Distro = distro,
|
||||
Release = release,
|
||||
SourcePkg = sourcePkg,
|
||||
CveId = cve,
|
||||
State = FixState.Fixed,
|
||||
FixedVersion = version,
|
||||
Method = FixMethod.Changelog,
|
||||
Confidence = 0.75m,
|
||||
Evidence = new ChangelogEvidence
|
||||
{
|
||||
File = "*.spec",
|
||||
Version = version,
|
||||
Excerpt = entryText.Length > 2000 ? entryText[..2000] : entryText,
|
||||
LineNumber = null
|
||||
},
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using StellaOps.BinaryIndex.FixIndex.Models;
|
||||
|
||||
namespace StellaOps.BinaryIndex.FixIndex.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// Repository interface for CVE fix index operations.
|
||||
/// </summary>
|
||||
public interface IFixIndexRepository
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the fix status for a specific CVE/package/distro combination.
|
||||
/// </summary>
|
||||
/// <param name="distro">Distribution (debian, ubuntu, alpine, rhel)</param>
|
||||
/// <param name="release">Release codename (bookworm, jammy, v3.19)</param>
|
||||
/// <param name="sourcePkg">Source package name</param>
|
||||
/// <param name="cveId">CVE identifier</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Fix status if found, null otherwise</returns>
|
||||
Task<FixIndexEntry?> GetFixStatusAsync(
|
||||
string distro,
|
||||
string release,
|
||||
string sourcePkg,
|
||||
string cveId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all fix statuses for a package.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<FixIndexEntry>> GetFixStatusesForPackageAsync(
|
||||
string distro,
|
||||
string release,
|
||||
string sourcePkg,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all known fix locations for a CVE across distros.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<FixIndexEntry>> GetFixLocationsForCveAsync(
|
||||
string cveId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Upserts a fix index entry.
|
||||
/// </summary>
|
||||
Task<FixIndexEntry> UpsertAsync(
|
||||
FixEvidence evidence,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Batch upserts fix index entries.
|
||||
/// </summary>
|
||||
Task<int> UpsertBatchAsync(
|
||||
IEnumerable<FixEvidence> evidenceList,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Stores fix evidence for audit trail.
|
||||
/// </summary>
|
||||
Task<Guid> StoreEvidenceAsync(
|
||||
FixEvidence evidence,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets evidence by ID.
|
||||
/// </summary>
|
||||
Task<FixEvidenceRecord?> GetEvidenceAsync(
|
||||
Guid evidenceId,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all entries from a specific snapshot (for re-ingestion).
|
||||
/// </summary>
|
||||
Task<int> DeleteBySnapshotAsync(
|
||||
Guid snapshotId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix index entry from the database.
|
||||
/// </summary>
|
||||
public sealed record FixIndexEntry
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
public required string Distro { get; init; }
|
||||
public required string Release { get; init; }
|
||||
public required string SourcePkg { get; init; }
|
||||
public required string CveId { get; init; }
|
||||
public required FixState State { get; init; }
|
||||
public string? FixedVersion { get; init; }
|
||||
public required FixMethod Method { get; init; }
|
||||
public required decimal Confidence { get; init; }
|
||||
public Guid? EvidenceId { get; init; }
|
||||
public Guid? SnapshotId { get; init; }
|
||||
public required DateTimeOffset IndexedAt { get; init; }
|
||||
public required DateTimeOffset UpdatedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix evidence record from the database.
|
||||
/// </summary>
|
||||
public sealed record FixEvidenceRecord
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
public required string EvidenceType { get; init; }
|
||||
public string? SourceFile { get; init; }
|
||||
public string? SourceSha256 { get; init; }
|
||||
public string? Excerpt { get; init; }
|
||||
public required string MetadataJson { get; init; }
|
||||
public Guid? SnapshotId { get; init; }
|
||||
public required DateTimeOffset CreatedAt { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.BinaryIndex.FixIndex.Models;
|
||||
using StellaOps.BinaryIndex.FixIndex.Parsers;
|
||||
|
||||
namespace StellaOps.BinaryIndex.FixIndex.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IFixIndexBuilder"/>.
|
||||
/// </summary>
|
||||
public sealed class FixIndexBuilder : IFixIndexBuilder
|
||||
{
|
||||
private readonly ILogger<FixIndexBuilder> _logger;
|
||||
private readonly DebianChangelogParser _debianParser;
|
||||
private readonly PatchHeaderParser _patchParser;
|
||||
private readonly AlpineSecfixesParser _alpineParser;
|
||||
private readonly RpmChangelogParser _rpmParser;
|
||||
|
||||
public FixIndexBuilder(ILogger<FixIndexBuilder> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_debianParser = new DebianChangelogParser();
|
||||
_patchParser = new PatchHeaderParser();
|
||||
_alpineParser = new AlpineSecfixesParser();
|
||||
_rpmParser = new RpmChangelogParser();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<FixEvidence> BuildDebianIndexAsync(
|
||||
DebianFixIndexRequest request,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Building Debian fix index for {Distro}/{Release}/{Package}",
|
||||
request.Distro, request.Release, request.SourcePkg);
|
||||
|
||||
var cvesSeen = new HashSet<string>();
|
||||
|
||||
// Parse changelog for CVE mentions
|
||||
if (!string.IsNullOrWhiteSpace(request.Changelog))
|
||||
{
|
||||
foreach (var evidence in _debianParser.ParseTopEntry(
|
||||
request.Changelog,
|
||||
request.Distro,
|
||||
request.Release,
|
||||
request.SourcePkg))
|
||||
{
|
||||
if (cvesSeen.Add(evidence.CveId))
|
||||
{
|
||||
yield return evidence with { SnapshotId = request.SnapshotId };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse patches for CVE mentions (DEP-3 format)
|
||||
if (request.Patches != null && request.Patches.Count > 0 && !string.IsNullOrEmpty(request.Version))
|
||||
{
|
||||
var patchTuples = request.Patches
|
||||
.Select(p => (p.Path, p.Content, p.Sha256));
|
||||
|
||||
foreach (var evidence in _patchParser.ParsePatches(
|
||||
patchTuples,
|
||||
request.Distro,
|
||||
request.Release,
|
||||
request.SourcePkg,
|
||||
request.Version))
|
||||
{
|
||||
// Patches have higher confidence, so they can override changelog entries
|
||||
if (cvesSeen.Add(evidence.CveId) || evidence.Confidence > 0.85m)
|
||||
{
|
||||
yield return evidence with { SnapshotId = request.SnapshotId };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await Task.CompletedTask; // Satisfy async requirement
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<FixEvidence> BuildAlpineIndexAsync(
|
||||
AlpineFixIndexRequest request,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Building Alpine fix index for {Release}/{Package}",
|
||||
request.Release, request.SourcePkg);
|
||||
|
||||
foreach (var evidence in _alpineParser.Parse(
|
||||
request.ApkBuild,
|
||||
request.Distro,
|
||||
request.Release,
|
||||
request.SourcePkg))
|
||||
{
|
||||
yield return evidence with { SnapshotId = request.SnapshotId };
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<FixEvidence> BuildRpmIndexAsync(
|
||||
RpmFixIndexRequest request,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Building RPM fix index for {Distro}/{Release}/{Package}",
|
||||
request.Distro, request.Release, request.SourcePkg);
|
||||
|
||||
// Parse spec file changelog
|
||||
foreach (var evidence in _rpmParser.ParseAllEntries(
|
||||
request.SpecContent,
|
||||
request.Distro,
|
||||
request.Release,
|
||||
request.SourcePkg))
|
||||
{
|
||||
yield return evidence with { SnapshotId = request.SnapshotId };
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
using StellaOps.BinaryIndex.FixIndex.Models;
|
||||
|
||||
namespace StellaOps.BinaryIndex.FixIndex.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for building the CVE fix index from various sources.
|
||||
/// </summary>
|
||||
public interface IFixIndexBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds fix index entries for a Debian/Ubuntu package.
|
||||
/// </summary>
|
||||
/// <param name="request">The Debian build request.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Fix evidence entries.</returns>
|
||||
IAsyncEnumerable<FixEvidence> BuildDebianIndexAsync(
|
||||
DebianFixIndexRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Builds fix index entries for an Alpine package.
|
||||
/// </summary>
|
||||
/// <param name="request">The Alpine build request.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Fix evidence entries.</returns>
|
||||
IAsyncEnumerable<FixEvidence> BuildAlpineIndexAsync(
|
||||
AlpineFixIndexRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Builds fix index entries for an RPM package.
|
||||
/// </summary>
|
||||
/// <param name="request">The RPM build request.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Fix evidence entries.</returns>
|
||||
IAsyncEnumerable<FixEvidence> BuildRpmIndexAsync(
|
||||
RpmFixIndexRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request for building Debian fix index.
|
||||
/// </summary>
|
||||
public sealed record DebianFixIndexRequest
|
||||
{
|
||||
/// <summary>Distribution (debian or ubuntu).</summary>
|
||||
public required string Distro { get; init; }
|
||||
|
||||
/// <summary>Release codename (bookworm, jammy).</summary>
|
||||
public required string Release { get; init; }
|
||||
|
||||
/// <summary>Source package name.</summary>
|
||||
public required string SourcePkg { get; init; }
|
||||
|
||||
/// <summary>Changelog content.</summary>
|
||||
public string? Changelog { get; init; }
|
||||
|
||||
/// <summary>Patches with path, content, and SHA-256.</summary>
|
||||
public IReadOnlyList<PatchFile>? Patches { get; init; }
|
||||
|
||||
/// <summary>Package version for patch association.</summary>
|
||||
public string? Version { get; init; }
|
||||
|
||||
/// <summary>Corpus snapshot ID.</summary>
|
||||
public Guid? SnapshotId { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request for building Alpine fix index.
|
||||
/// </summary>
|
||||
public sealed record AlpineFixIndexRequest
|
||||
{
|
||||
/// <summary>Distribution (always "alpine").</summary>
|
||||
public string Distro => "alpine";
|
||||
|
||||
/// <summary>Release (v3.19, edge).</summary>
|
||||
public required string Release { get; init; }
|
||||
|
||||
/// <summary>Source package name.</summary>
|
||||
public required string SourcePkg { get; init; }
|
||||
|
||||
/// <summary>APKBUILD file content.</summary>
|
||||
public required string ApkBuild { get; init; }
|
||||
|
||||
/// <summary>Corpus snapshot ID.</summary>
|
||||
public Guid? SnapshotId { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request for building RPM fix index.
|
||||
/// </summary>
|
||||
public sealed record RpmFixIndexRequest
|
||||
{
|
||||
/// <summary>Distribution (rhel, fedora, centos, rocky, alma).</summary>
|
||||
public required string Distro { get; init; }
|
||||
|
||||
/// <summary>Release version (9, 39, etc.).</summary>
|
||||
public required string Release { get; init; }
|
||||
|
||||
/// <summary>Source package name.</summary>
|
||||
public required string SourcePkg { get; init; }
|
||||
|
||||
/// <summary>Spec file content.</summary>
|
||||
public required string SpecContent { get; init; }
|
||||
|
||||
/// <summary>Corpus snapshot ID.</summary>
|
||||
public Guid? SnapshotId { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a patch file with content.
|
||||
/// </summary>
|
||||
public sealed record PatchFile
|
||||
{
|
||||
/// <summary>Relative path to the patch file.</summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>Content of the patch file.</summary>
|
||||
public required string Content { get; init; }
|
||||
|
||||
/// <summary>SHA-256 hash of the patch content.</summary>
|
||||
public required string Sha256 { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user