Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly.
- Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps.
- Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges.
- Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges.
- Set up project file for the test project with necessary dependencies and configurations.
- Include JSON fixture files for testing purposes.
This commit is contained in:
master
2025-11-04 07:49:39 +02:00
parent f72c5c513a
commit 2eb6852d34
491 changed files with 39445 additions and 3917 deletions

View File

@@ -23,21 +23,23 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Connector.Ics.Cisa.Configuration;
using StellaOps.Concelier.Connector.Ics.Cisa.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Normalization.SemVer;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Ics.Cisa;
public sealed class IcsCisaConnector : IFeedConnector
{
private const string SchemaVersion = "ics.cisa.feed.v1";
private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };
private const string SchemaVersion = "ics.cisa.feed.v1";
private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };
private static readonly Regex FirmwareRangeRegex = new(@"(?<range>(?:<=?|>=?)?\s*\d+(?:\.\d+){0,2}(?:\s*-\s*\d+(?:\.\d+){0,2})?)", RegexOptions.CultureInvariant);
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
@@ -653,51 +655,46 @@ public sealed class IcsCisaConnector : IFeedConnector
.Where(static product => !string.IsNullOrWhiteSpace(product.Name))
.ToArray();
if (parsedProducts.Length > 0)
{
foreach (var product in parsedProducts)
{
}
foreach (var product in parsedProducts)
{
var provenance = new AdvisoryProvenance("ics-cisa", "affected", product.Name!, recordedAt);
var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["ics.product"] = product.Name!
};
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
{
vendorExtensions["ics.version"] = product.VersionExpression!;
}
if (normalizedVendors.Length > 0)
{
vendorExtensions["ics.vendors"] = string.Join(",", normalizedVendors);
}
var semVer = TryCreateSemVerPrimitive(product.VersionExpression);
var range = new AffectedVersionRange(
rangeKind: "product",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.VersionExpression,
provenance: provenance,
primitives: new RangePrimitives(semVer, null, null, vendorExtensions));
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
product.Name!,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance }));
}
return packages;
}
if (parsedProducts.Length > 0)
{
for (var index = 0; index < parsedProducts.Length; index++)
{
var product = parsedProducts[index];
var provenanceKey = BuildProvenanceKey(advisoryDto.AdvisoryId, product.Name, index);
var vendorExtensions = CreateVendorExtensions(product, normalizedVendors);
var (ranges, normalizedRules) = BuildVersionArtifacts(product, provenanceKey, recordedAt, vendorExtensions);
var fieldMasks = new List<string> { ProvenanceFieldMasks.AffectedPackages };
if (ranges.Count > 0)
{
fieldMasks.Add(ProvenanceFieldMasks.VersionRanges);
}
if (normalizedRules.Count > 0)
{
fieldMasks.Add(ProvenanceFieldMasks.NormalizedVersions);
}
var packageProvenance = new AdvisoryProvenance(
"ics-cisa",
"affected",
provenanceKey,
recordedAt,
fieldMasks);
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
product.Name!,
platform: null,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { packageProvenance },
normalizedVersions: normalizedRules));
}
return packages;
}
if (normalizedVendors.Length == 0)
{
@@ -721,32 +718,48 @@ public sealed class IcsCisaConnector : IFeedConnector
provenance: provenance,
primitives: new RangePrimitives(null, null, null, vendorExtensions));
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
vendor,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance }));
}
return packages;
}
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
vendor,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance }));
}
return packages;
}
private static ProductInfo ParseProductInfo(string raw)
{
var trimmed = raw?.Trim();
if (string.IsNullOrWhiteSpace(trimmed))
{
return new ProductInfo(null, null);
}
if (trimmed.Contains(':', StringComparison.Ordinal))
{
var parts = trimmed.Split(':', 2);
var name = parts[0].Trim();
var versionSegment = parts[1].Trim();
private static ProductInfo ParseProductInfo(string raw)
{
var trimmed = raw?.Trim();
if (string.IsNullOrWhiteSpace(trimmed))
{
return new ProductInfo(null, null);
}
var rangeMatch = FirmwareRangeRegex.Match(trimmed);
if (rangeMatch.Success)
{
var range = rangeMatch.Groups["range"].Value.Trim();
if (!string.IsNullOrEmpty(range))
{
var withoutRange = trimmed.Remove(rangeMatch.Index, rangeMatch.Length).TrimEnd('-', ':', ';', ',', '.', ' ');
if (string.IsNullOrWhiteSpace(withoutRange))
{
withoutRange = trimmed;
}
return new ProductInfo(withoutRange, range);
}
}
if (trimmed.Contains(':', StringComparison.Ordinal))
{
var parts = trimmed.Split(':', 2);
var name = parts[0].Trim();
var versionSegment = parts[1].Trim();
return new ProductInfo(
string.IsNullOrWhiteSpace(name) ? trimmed : name,
string.IsNullOrWhiteSpace(versionSegment) ? null : versionSegment);
@@ -811,11 +824,11 @@ public sealed class IcsCisaConnector : IFeedConnector
normalized);
}
private static string? NormalizeSemVer(string rawVersion)
{
var trimmed = rawVersion.Trim();
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
{
private static string? NormalizeSemVer(string rawVersion)
{
var trimmed = rawVersion.Trim();
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed[1..];
}
@@ -830,11 +843,169 @@ public sealed class IcsCisaConnector : IFeedConnector
{
components.Add("0");
}
return string.Join('.', components);
}
private sealed record ProductInfo(string? Name, string? VersionExpression);
return string.Join('.', components);
}
private static string BuildProvenanceKey(string advisoryId, string? productName, int index)
{
var slug = Slugify(productName);
if (string.IsNullOrEmpty(slug))
{
slug = (index + 1).ToString(CultureInfo.InvariantCulture);
}
return $"ics-cisa:{advisoryId}:{slug}";
}
private static string Slugify(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var builder = new StringBuilder(value.Length);
foreach (var ch in value.ToLowerInvariant())
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(ch);
}
else if (builder.Length > 0 && builder[^1] != '-')
{
builder.Append('-');
}
}
return builder.ToString().Trim('-');
}
private static IReadOnlyDictionary<string, string> CreateVendorExtensions(ProductInfo product, IReadOnlyList<string> normalizedVendors)
{
var extensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["ics.product"] = product.Name!
};
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
{
extensions["ics.version"] = product.VersionExpression!;
}
if (normalizedVendors.Count > 0)
{
extensions["ics.vendors"] = string.Join(",", normalizedVendors);
}
return extensions;
}
private static (List<AffectedVersionRange> Ranges, List<NormalizedVersionRule> NormalizedRules) BuildVersionArtifacts(
ProductInfo product,
string provenanceKey,
DateTimeOffset recordedAt,
IReadOnlyDictionary<string, string> vendorExtensions)
{
var ranges = new List<AffectedVersionRange>();
var normalizedRules = new List<NormalizedVersionRule>();
var rangeProvenance = new AdvisoryProvenance(
"ics-cisa",
"affected.version",
provenanceKey,
recordedAt,
new[]
{
ProvenanceFieldMasks.VersionRanges
});
var semverResults = string.IsNullOrWhiteSpace(product.VersionExpression)
? Array.Empty<SemVerRangeBuildResult>()
: SemVerRangeRuleBuilder.Build(product.VersionExpression, provenanceNote: provenanceKey);
if (semverResults.Count > 0)
{
foreach (var result in semverResults)
{
var rangeExtensions = CloneVendorExtensions(vendorExtensions);
var rawExpression = string.IsNullOrWhiteSpace(product.VersionExpression)
? result.Expression
: product.VersionExpression!.Trim();
rangeExtensions["ics.range.expression"] = rawExpression;
rangeExtensions["ics.range.normalized"] = result.Expression;
ranges.Add(new AffectedVersionRange(
rangeKind: "product",
introducedVersion: result.Primitive.Introduced,
fixedVersion: result.Primitive.Fixed,
lastAffectedVersion: result.Primitive.LastAffected,
rangeExpression: rawExpression,
provenance: rangeProvenance,
primitives: new RangePrimitives(result.Primitive, null, null, rangeExtensions)));
normalizedRules.Add(result.NormalizedRule);
}
return (ranges, normalizedRules);
}
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
{
var primitive = TryCreateSemVerPrimitive(product.VersionExpression);
if (primitive is not null)
{
var expression = primitive.ConstraintExpression ?? product.VersionExpression!.Trim();
var rangeExtensions = CloneVendorExtensions(vendorExtensions);
rangeExtensions["ics.range.expression"] = expression;
ranges.Add(new AffectedVersionRange(
rangeKind: "product",
introducedVersion: primitive.Introduced,
fixedVersion: primitive.Fixed,
lastAffectedVersion: primitive.LastAffected,
rangeExpression: expression,
provenance: rangeProvenance,
primitives: new RangePrimitives(primitive, null, null, rangeExtensions)));
var normalizedRule = primitive.ToNormalizedVersionRule(provenanceKey);
if (normalizedRule is not null)
{
normalizedRules.Add(normalizedRule);
}
return (ranges, normalizedRules);
}
}
var fallbackExtensions = CloneVendorExtensions(vendorExtensions);
ranges.Add(new AffectedVersionRange(
rangeKind: "product",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.VersionExpression,
provenance: rangeProvenance,
primitives: new RangePrimitives(null, null, null, fallbackExtensions)));
return (ranges, normalizedRules);
}
private static Dictionary<string, string> CloneVendorExtensions(IReadOnlyDictionary<string, string> source)
{
var clone = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var pair in source)
{
if (!string.IsNullOrWhiteSpace(pair.Key) && pair.Value is not null)
{
clone[pair.Key] = pair.Value;
}
}
return clone;
}
private sealed record ProductInfo(string? Name, string? VersionExpression);
private async Task<IcsCisaAdvisoryDto> EnrichAdvisoryAsync(IcsCisaAdvisoryDto advisory, CancellationToken cancellationToken)
{

View File

@@ -13,6 +13,7 @@
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
</ItemGroup>
@@ -26,4 +27,4 @@
</AssemblyAttribute>
</ItemGroup>
</Project>
</Project>

View File

@@ -1,4 +1,4 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|FEEDCONN-ICSCISA-02-012 Version range provenance|BE-Conn-ICS-CISA|CONCELIER-LNM-21-001|**TODO (due 2025-10-23)** Promote existing firmware/semver data into `advisory_observations.affected.versions[]` entries with deterministic comparison keys and provenance identifiers (`ics-cisa:{advisoryId}:{product}`). Add regression coverage for mixed firmware strings and raise a Models ticket only when observation schema needs a new comparison helper.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to build observation version entries and log failures without invoking the retired merge helpers.|
|FEEDCONN-ICSCISA-02-012 Version range provenance|BE-Conn-ICS-CISA|CONCELIER-LNM-21-001|**DONE (2025-11-03)** Promote existing firmware/semver data into `advisory_observations.affected.versions[]` entries with deterministic comparison keys and provenance identifiers (`ics-cisa:{advisoryId}:{product}`). Add regression coverage for mixed firmware strings and raise a Models ticket only when observation schema needs a new comparison helper.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to build observation version entries and log failures without invoking the retired merge helpers.<br>2025-11-03: Completed connector now emits semver-aware range rules with provenance, RSS fallback payloads pass the guard, and Fetch/Parse/Map end-to-end coverage succeeds.|