Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly. - Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps. - Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges. - Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges. - Set up project file for the test project with necessary dependencies and configurations. - Include JSON fixture files for testing purposes.
This commit is contained in:
@@ -23,21 +23,23 @@ using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
using StellaOps.Concelier.Connector.Ics.Cisa.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ics.Cisa.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Normalization.SemVer;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Cisa;
|
||||
|
||||
public sealed class IcsCisaConnector : IFeedConnector
|
||||
{
|
||||
private const string SchemaVersion = "ics.cisa.feed.v1";
|
||||
|
||||
private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
|
||||
private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
|
||||
private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };
|
||||
private const string SchemaVersion = "ics.cisa.feed.v1";
|
||||
|
||||
private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
|
||||
private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
|
||||
private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };
|
||||
private static readonly Regex FirmwareRangeRegex = new(@"(?<range>(?:<=?|>=?)?\s*\d+(?:\.\d+){0,2}(?:\s*-\s*\d+(?:\.\d+){0,2})?)", RegexOptions.CultureInvariant);
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
@@ -653,51 +655,46 @@ public sealed class IcsCisaConnector : IFeedConnector
|
||||
.Where(static product => !string.IsNullOrWhiteSpace(product.Name))
|
||||
.ToArray();
|
||||
|
||||
if (parsedProducts.Length > 0)
|
||||
{
|
||||
foreach (var product in parsedProducts)
|
||||
{
|
||||
}
|
||||
|
||||
foreach (var product in parsedProducts)
|
||||
{
|
||||
var provenance = new AdvisoryProvenance("ics-cisa", "affected", product.Name!, recordedAt);
|
||||
var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["ics.product"] = product.Name!
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
|
||||
{
|
||||
vendorExtensions["ics.version"] = product.VersionExpression!;
|
||||
}
|
||||
|
||||
if (normalizedVendors.Length > 0)
|
||||
{
|
||||
vendorExtensions["ics.vendors"] = string.Join(",", normalizedVendors);
|
||||
}
|
||||
|
||||
var semVer = TryCreateSemVerPrimitive(product.VersionExpression);
|
||||
var range = new AffectedVersionRange(
|
||||
rangeKind: "product",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.VersionExpression,
|
||||
provenance: provenance,
|
||||
primitives: new RangePrimitives(semVer, null, null, vendorExtensions));
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.IcsVendor,
|
||||
product.Name!,
|
||||
platform: null,
|
||||
versionRanges: new[] { range },
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance }));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
if (parsedProducts.Length > 0)
|
||||
{
|
||||
for (var index = 0; index < parsedProducts.Length; index++)
|
||||
{
|
||||
var product = parsedProducts[index];
|
||||
var provenanceKey = BuildProvenanceKey(advisoryDto.AdvisoryId, product.Name, index);
|
||||
|
||||
var vendorExtensions = CreateVendorExtensions(product, normalizedVendors);
|
||||
var (ranges, normalizedRules) = BuildVersionArtifacts(product, provenanceKey, recordedAt, vendorExtensions);
|
||||
|
||||
var fieldMasks = new List<string> { ProvenanceFieldMasks.AffectedPackages };
|
||||
if (ranges.Count > 0)
|
||||
{
|
||||
fieldMasks.Add(ProvenanceFieldMasks.VersionRanges);
|
||||
}
|
||||
|
||||
if (normalizedRules.Count > 0)
|
||||
{
|
||||
fieldMasks.Add(ProvenanceFieldMasks.NormalizedVersions);
|
||||
}
|
||||
|
||||
var packageProvenance = new AdvisoryProvenance(
|
||||
"ics-cisa",
|
||||
"affected",
|
||||
provenanceKey,
|
||||
recordedAt,
|
||||
fieldMasks);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.IcsVendor,
|
||||
product.Name!,
|
||||
platform: null,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { packageProvenance },
|
||||
normalizedVersions: normalizedRules));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
if (normalizedVendors.Length == 0)
|
||||
{
|
||||
@@ -721,32 +718,48 @@ public sealed class IcsCisaConnector : IFeedConnector
|
||||
provenance: provenance,
|
||||
primitives: new RangePrimitives(null, null, null, vendorExtensions));
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.IcsVendor,
|
||||
vendor,
|
||||
platform: null,
|
||||
versionRanges: new[] { range },
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance }));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.IcsVendor,
|
||||
vendor,
|
||||
platform: null,
|
||||
versionRanges: new[] { range },
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance }));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
|
||||
private static ProductInfo ParseProductInfo(string raw)
|
||||
{
|
||||
var trimmed = raw?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return new ProductInfo(null, null);
|
||||
}
|
||||
|
||||
if (trimmed.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
var parts = trimmed.Split(':', 2);
|
||||
var name = parts[0].Trim();
|
||||
var versionSegment = parts[1].Trim();
|
||||
private static ProductInfo ParseProductInfo(string raw)
|
||||
{
|
||||
var trimmed = raw?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return new ProductInfo(null, null);
|
||||
}
|
||||
|
||||
var rangeMatch = FirmwareRangeRegex.Match(trimmed);
|
||||
if (rangeMatch.Success)
|
||||
{
|
||||
var range = rangeMatch.Groups["range"].Value.Trim();
|
||||
if (!string.IsNullOrEmpty(range))
|
||||
{
|
||||
var withoutRange = trimmed.Remove(rangeMatch.Index, rangeMatch.Length).TrimEnd('-', ':', ';', ',', '.', ' ');
|
||||
if (string.IsNullOrWhiteSpace(withoutRange))
|
||||
{
|
||||
withoutRange = trimmed;
|
||||
}
|
||||
|
||||
return new ProductInfo(withoutRange, range);
|
||||
}
|
||||
}
|
||||
|
||||
if (trimmed.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
var parts = trimmed.Split(':', 2);
|
||||
var name = parts[0].Trim();
|
||||
var versionSegment = parts[1].Trim();
|
||||
return new ProductInfo(
|
||||
string.IsNullOrWhiteSpace(name) ? trimmed : name,
|
||||
string.IsNullOrWhiteSpace(versionSegment) ? null : versionSegment);
|
||||
@@ -811,11 +824,11 @@ public sealed class IcsCisaConnector : IFeedConnector
|
||||
normalized);
|
||||
}
|
||||
|
||||
private static string? NormalizeSemVer(string rawVersion)
|
||||
{
|
||||
var trimmed = rawVersion.Trim();
|
||||
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
private static string? NormalizeSemVer(string rawVersion)
|
||||
{
|
||||
var trimmed = rawVersion.Trim();
|
||||
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
trimmed = trimmed[1..];
|
||||
}
|
||||
|
||||
@@ -830,11 +843,169 @@ public sealed class IcsCisaConnector : IFeedConnector
|
||||
{
|
||||
components.Add("0");
|
||||
}
|
||||
|
||||
return string.Join('.', components);
|
||||
}
|
||||
|
||||
private sealed record ProductInfo(string? Name, string? VersionExpression);
|
||||
|
||||
return string.Join('.', components);
|
||||
}
|
||||
|
||||
private static string BuildProvenanceKey(string advisoryId, string? productName, int index)
|
||||
{
|
||||
var slug = Slugify(productName);
|
||||
if (string.IsNullOrEmpty(slug))
|
||||
{
|
||||
slug = (index + 1).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
return $"ics-cisa:{advisoryId}:{slug}";
|
||||
}
|
||||
|
||||
private static string Slugify(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(value.Length);
|
||||
foreach (var ch in value.ToLowerInvariant())
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
builder.Append(ch);
|
||||
}
|
||||
else if (builder.Length > 0 && builder[^1] != '-')
|
||||
{
|
||||
builder.Append('-');
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToString().Trim('-');
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> CreateVendorExtensions(ProductInfo product, IReadOnlyList<string> normalizedVendors)
|
||||
{
|
||||
var extensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["ics.product"] = product.Name!
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
|
||||
{
|
||||
extensions["ics.version"] = product.VersionExpression!;
|
||||
}
|
||||
|
||||
if (normalizedVendors.Count > 0)
|
||||
{
|
||||
extensions["ics.vendors"] = string.Join(",", normalizedVendors);
|
||||
}
|
||||
|
||||
return extensions;
|
||||
}
|
||||
|
||||
private static (List<AffectedVersionRange> Ranges, List<NormalizedVersionRule> NormalizedRules) BuildVersionArtifacts(
|
||||
ProductInfo product,
|
||||
string provenanceKey,
|
||||
DateTimeOffset recordedAt,
|
||||
IReadOnlyDictionary<string, string> vendorExtensions)
|
||||
{
|
||||
var ranges = new List<AffectedVersionRange>();
|
||||
var normalizedRules = new List<NormalizedVersionRule>();
|
||||
|
||||
var rangeProvenance = new AdvisoryProvenance(
|
||||
"ics-cisa",
|
||||
"affected.version",
|
||||
provenanceKey,
|
||||
recordedAt,
|
||||
new[]
|
||||
{
|
||||
ProvenanceFieldMasks.VersionRanges
|
||||
});
|
||||
|
||||
var semverResults = string.IsNullOrWhiteSpace(product.VersionExpression)
|
||||
? Array.Empty<SemVerRangeBuildResult>()
|
||||
: SemVerRangeRuleBuilder.Build(product.VersionExpression, provenanceNote: provenanceKey);
|
||||
|
||||
if (semverResults.Count > 0)
|
||||
{
|
||||
foreach (var result in semverResults)
|
||||
{
|
||||
var rangeExtensions = CloneVendorExtensions(vendorExtensions);
|
||||
var rawExpression = string.IsNullOrWhiteSpace(product.VersionExpression)
|
||||
? result.Expression
|
||||
: product.VersionExpression!.Trim();
|
||||
rangeExtensions["ics.range.expression"] = rawExpression;
|
||||
rangeExtensions["ics.range.normalized"] = result.Expression;
|
||||
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "product",
|
||||
introducedVersion: result.Primitive.Introduced,
|
||||
fixedVersion: result.Primitive.Fixed,
|
||||
lastAffectedVersion: result.Primitive.LastAffected,
|
||||
rangeExpression: rawExpression,
|
||||
provenance: rangeProvenance,
|
||||
primitives: new RangePrimitives(result.Primitive, null, null, rangeExtensions)));
|
||||
|
||||
normalizedRules.Add(result.NormalizedRule);
|
||||
}
|
||||
|
||||
return (ranges, normalizedRules);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
|
||||
{
|
||||
var primitive = TryCreateSemVerPrimitive(product.VersionExpression);
|
||||
if (primitive is not null)
|
||||
{
|
||||
var expression = primitive.ConstraintExpression ?? product.VersionExpression!.Trim();
|
||||
var rangeExtensions = CloneVendorExtensions(vendorExtensions);
|
||||
rangeExtensions["ics.range.expression"] = expression;
|
||||
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "product",
|
||||
introducedVersion: primitive.Introduced,
|
||||
fixedVersion: primitive.Fixed,
|
||||
lastAffectedVersion: primitive.LastAffected,
|
||||
rangeExpression: expression,
|
||||
provenance: rangeProvenance,
|
||||
primitives: new RangePrimitives(primitive, null, null, rangeExtensions)));
|
||||
|
||||
var normalizedRule = primitive.ToNormalizedVersionRule(provenanceKey);
|
||||
if (normalizedRule is not null)
|
||||
{
|
||||
normalizedRules.Add(normalizedRule);
|
||||
}
|
||||
|
||||
return (ranges, normalizedRules);
|
||||
}
|
||||
}
|
||||
|
||||
var fallbackExtensions = CloneVendorExtensions(vendorExtensions);
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "product",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.VersionExpression,
|
||||
provenance: rangeProvenance,
|
||||
primitives: new RangePrimitives(null, null, null, fallbackExtensions)));
|
||||
|
||||
return (ranges, normalizedRules);
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> CloneVendorExtensions(IReadOnlyDictionary<string, string> source)
|
||||
{
|
||||
var clone = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var pair in source)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(pair.Key) && pair.Value is not null)
|
||||
{
|
||||
clone[pair.Key] = pair.Value;
|
||||
}
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
private sealed record ProductInfo(string? Name, string? VersionExpression);
|
||||
|
||||
private async Task<IcsCisaAdvisoryDto> EnrichAdvisoryAsync(IcsCisaAdvisoryDto advisory, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -26,4 +27,4 @@
|
||||
</AssemblyAttribute>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-ICSCISA-02-012 Version range provenance|BE-Conn-ICS-CISA|CONCELIER-LNM-21-001|**TODO (due 2025-10-23)** – Promote existing firmware/semver data into `advisory_observations.affected.versions[]` entries with deterministic comparison keys and provenance identifiers (`ics-cisa:{advisoryId}:{product}`). Add regression coverage for mixed firmware strings and raise a Models ticket only when observation schema needs a new comparison helper.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to build observation version entries and log failures without invoking the retired merge helpers.|
|
||||
|FEEDCONN-ICSCISA-02-012 Version range provenance|BE-Conn-ICS-CISA|CONCELIER-LNM-21-001|**DONE (2025-11-03)** – Promote existing firmware/semver data into `advisory_observations.affected.versions[]` entries with deterministic comparison keys and provenance identifiers (`ics-cisa:{advisoryId}:{product}`). Add regression coverage for mixed firmware strings and raise a Models ticket only when observation schema needs a new comparison helper.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to build observation version entries and log failures without invoking the retired merge helpers.<br>2025-11-03: Completed – connector now emits semver-aware range rules with provenance, RSS fallback payloads pass the guard, and Fetch/Parse/Map end-to-end coverage succeeds.|
|
||||
|
||||
Reference in New Issue
Block a user