Add unit tests for SBOM ingestion and transformation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implement `SbomIngestServiceCollectionExtensionsTests` to verify the SBOM ingestion pipeline exports snapshots correctly.
- Create `SbomIngestTransformerTests` to ensure the transformation produces expected nodes and edges, including deduplication of license nodes and normalization of timestamps.
- Add `SbomSnapshotExporterTests` to test the export functionality for manifest, adjacency, nodes, and edges.
- Introduce `VexOverlayTransformerTests` to validate the transformation of VEX nodes and edges.
- Set up project file for the test project with necessary dependencies and configurations.
- Include JSON fixture files for testing purposes.
This commit is contained in:
master
2025-11-04 07:49:39 +02:00
parent f72c5c513a
commit 2eb6852d34
491 changed files with 39445 additions and 3917 deletions

View File

@@ -1,3 +1,4 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
@@ -298,10 +299,44 @@ public sealed class SourceFetchService
}
catch (JsonException ex)
{
throw new InvalidOperationException($"Raw advisory payload from {request.SourceName} is not valid JSON ({request.RequestUri}).", ex);
var fallbackDocument = CreateFallbackContentDocument(request, contentBytes, ex);
return fallbackDocument;
}
}
private static JsonDocument CreateFallbackContentDocument(
SourceFetchRequest request,
byte[] contentBytes,
JsonException parseException)
{
var payload = new Dictionary<string, object?>
{
["type"] = "non-json",
["encoding"] = "base64",
["source"] = request.SourceName,
["uri"] = request.RequestUri.ToString(),
["mediaTypeHint"] = request.AcceptHeaders?.FirstOrDefault(),
["parseError"] = parseException.Message,
["raw"] = Convert.ToBase64String(contentBytes),
};
try
{
var text = Encoding.UTF8.GetString(contentBytes);
if (!string.IsNullOrWhiteSpace(text))
{
payload["text"] = text;
}
}
catch
{
// ignore decoding failures; base64 field already present
}
var buffer = JsonSerializer.SerializeToUtf8Bytes(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web));
return JsonDocument.Parse(buffer);
}
private static ImmutableDictionary<string, string> BuildProvenance(
SourceFetchRequest request,
HttpResponseMessage response,

View File

@@ -23,21 +23,23 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Connector.Ics.Cisa.Configuration;
using StellaOps.Concelier.Connector.Ics.Cisa.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Normalization.SemVer;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Ics.Cisa;
public sealed class IcsCisaConnector : IFeedConnector
{
private const string SchemaVersion = "ics.cisa.feed.v1";
private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };
private const string SchemaVersion = "ics.cisa.feed.v1";
private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };
private static readonly Regex FirmwareRangeRegex = new(@"(?<range>(?:<=?|>=?)?\s*\d+(?:\.\d+){0,2}(?:\s*-\s*\d+(?:\.\d+){0,2})?)", RegexOptions.CultureInvariant);
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
@@ -653,51 +655,46 @@ public sealed class IcsCisaConnector : IFeedConnector
.Where(static product => !string.IsNullOrWhiteSpace(product.Name))
.ToArray();
if (parsedProducts.Length > 0)
{
foreach (var product in parsedProducts)
{
}
foreach (var product in parsedProducts)
{
var provenance = new AdvisoryProvenance("ics-cisa", "affected", product.Name!, recordedAt);
var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["ics.product"] = product.Name!
};
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
{
vendorExtensions["ics.version"] = product.VersionExpression!;
}
if (normalizedVendors.Length > 0)
{
vendorExtensions["ics.vendors"] = string.Join(",", normalizedVendors);
}
var semVer = TryCreateSemVerPrimitive(product.VersionExpression);
var range = new AffectedVersionRange(
rangeKind: "product",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.VersionExpression,
provenance: provenance,
primitives: new RangePrimitives(semVer, null, null, vendorExtensions));
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
product.Name!,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance }));
}
return packages;
}
if (parsedProducts.Length > 0)
{
for (var index = 0; index < parsedProducts.Length; index++)
{
var product = parsedProducts[index];
var provenanceKey = BuildProvenanceKey(advisoryDto.AdvisoryId, product.Name, index);
var vendorExtensions = CreateVendorExtensions(product, normalizedVendors);
var (ranges, normalizedRules) = BuildVersionArtifacts(product, provenanceKey, recordedAt, vendorExtensions);
var fieldMasks = new List<string> { ProvenanceFieldMasks.AffectedPackages };
if (ranges.Count > 0)
{
fieldMasks.Add(ProvenanceFieldMasks.VersionRanges);
}
if (normalizedRules.Count > 0)
{
fieldMasks.Add(ProvenanceFieldMasks.NormalizedVersions);
}
var packageProvenance = new AdvisoryProvenance(
"ics-cisa",
"affected",
provenanceKey,
recordedAt,
fieldMasks);
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
product.Name!,
platform: null,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { packageProvenance },
normalizedVersions: normalizedRules));
}
return packages;
}
if (normalizedVendors.Length == 0)
{
@@ -721,32 +718,48 @@ public sealed class IcsCisaConnector : IFeedConnector
provenance: provenance,
primitives: new RangePrimitives(null, null, null, vendorExtensions));
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
vendor,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance }));
}
return packages;
}
packages.Add(new AffectedPackage(
AffectedPackageTypes.IcsVendor,
vendor,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance }));
}
return packages;
}
private static ProductInfo ParseProductInfo(string raw)
{
var trimmed = raw?.Trim();
if (string.IsNullOrWhiteSpace(trimmed))
{
return new ProductInfo(null, null);
}
if (trimmed.Contains(':', StringComparison.Ordinal))
{
var parts = trimmed.Split(':', 2);
var name = parts[0].Trim();
var versionSegment = parts[1].Trim();
private static ProductInfo ParseProductInfo(string raw)
{
var trimmed = raw?.Trim();
if (string.IsNullOrWhiteSpace(trimmed))
{
return new ProductInfo(null, null);
}
var rangeMatch = FirmwareRangeRegex.Match(trimmed);
if (rangeMatch.Success)
{
var range = rangeMatch.Groups["range"].Value.Trim();
if (!string.IsNullOrEmpty(range))
{
var withoutRange = trimmed.Remove(rangeMatch.Index, rangeMatch.Length).TrimEnd('-', ':', ';', ',', '.', ' ');
if (string.IsNullOrWhiteSpace(withoutRange))
{
withoutRange = trimmed;
}
return new ProductInfo(withoutRange, range);
}
}
if (trimmed.Contains(':', StringComparison.Ordinal))
{
var parts = trimmed.Split(':', 2);
var name = parts[0].Trim();
var versionSegment = parts[1].Trim();
return new ProductInfo(
string.IsNullOrWhiteSpace(name) ? trimmed : name,
string.IsNullOrWhiteSpace(versionSegment) ? null : versionSegment);
@@ -811,11 +824,11 @@ public sealed class IcsCisaConnector : IFeedConnector
normalized);
}
private static string? NormalizeSemVer(string rawVersion)
{
var trimmed = rawVersion.Trim();
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
{
private static string? NormalizeSemVer(string rawVersion)
{
var trimmed = rawVersion.Trim();
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed[1..];
}
@@ -830,11 +843,169 @@ public sealed class IcsCisaConnector : IFeedConnector
{
components.Add("0");
}
return string.Join('.', components);
}
private sealed record ProductInfo(string? Name, string? VersionExpression);
return string.Join('.', components);
}
private static string BuildProvenanceKey(string advisoryId, string? productName, int index)
{
var slug = Slugify(productName);
if (string.IsNullOrEmpty(slug))
{
slug = (index + 1).ToString(CultureInfo.InvariantCulture);
}
return $"ics-cisa:{advisoryId}:{slug}";
}
private static string Slugify(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var builder = new StringBuilder(value.Length);
foreach (var ch in value.ToLowerInvariant())
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(ch);
}
else if (builder.Length > 0 && builder[^1] != '-')
{
builder.Append('-');
}
}
return builder.ToString().Trim('-');
}
private static IReadOnlyDictionary<string, string> CreateVendorExtensions(ProductInfo product, IReadOnlyList<string> normalizedVendors)
{
var extensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["ics.product"] = product.Name!
};
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
{
extensions["ics.version"] = product.VersionExpression!;
}
if (normalizedVendors.Count > 0)
{
extensions["ics.vendors"] = string.Join(",", normalizedVendors);
}
return extensions;
}
private static (List<AffectedVersionRange> Ranges, List<NormalizedVersionRule> NormalizedRules) BuildVersionArtifacts(
ProductInfo product,
string provenanceKey,
DateTimeOffset recordedAt,
IReadOnlyDictionary<string, string> vendorExtensions)
{
var ranges = new List<AffectedVersionRange>();
var normalizedRules = new List<NormalizedVersionRule>();
var rangeProvenance = new AdvisoryProvenance(
"ics-cisa",
"affected.version",
provenanceKey,
recordedAt,
new[]
{
ProvenanceFieldMasks.VersionRanges
});
var semverResults = string.IsNullOrWhiteSpace(product.VersionExpression)
? Array.Empty<SemVerRangeBuildResult>()
: SemVerRangeRuleBuilder.Build(product.VersionExpression, provenanceNote: provenanceKey);
if (semverResults.Count > 0)
{
foreach (var result in semverResults)
{
var rangeExtensions = CloneVendorExtensions(vendorExtensions);
var rawExpression = string.IsNullOrWhiteSpace(product.VersionExpression)
? result.Expression
: product.VersionExpression!.Trim();
rangeExtensions["ics.range.expression"] = rawExpression;
rangeExtensions["ics.range.normalized"] = result.Expression;
ranges.Add(new AffectedVersionRange(
rangeKind: "product",
introducedVersion: result.Primitive.Introduced,
fixedVersion: result.Primitive.Fixed,
lastAffectedVersion: result.Primitive.LastAffected,
rangeExpression: rawExpression,
provenance: rangeProvenance,
primitives: new RangePrimitives(result.Primitive, null, null, rangeExtensions)));
normalizedRules.Add(result.NormalizedRule);
}
return (ranges, normalizedRules);
}
if (!string.IsNullOrWhiteSpace(product.VersionExpression))
{
var primitive = TryCreateSemVerPrimitive(product.VersionExpression);
if (primitive is not null)
{
var expression = primitive.ConstraintExpression ?? product.VersionExpression!.Trim();
var rangeExtensions = CloneVendorExtensions(vendorExtensions);
rangeExtensions["ics.range.expression"] = expression;
ranges.Add(new AffectedVersionRange(
rangeKind: "product",
introducedVersion: primitive.Introduced,
fixedVersion: primitive.Fixed,
lastAffectedVersion: primitive.LastAffected,
rangeExpression: expression,
provenance: rangeProvenance,
primitives: new RangePrimitives(primitive, null, null, rangeExtensions)));
var normalizedRule = primitive.ToNormalizedVersionRule(provenanceKey);
if (normalizedRule is not null)
{
normalizedRules.Add(normalizedRule);
}
return (ranges, normalizedRules);
}
}
var fallbackExtensions = CloneVendorExtensions(vendorExtensions);
ranges.Add(new AffectedVersionRange(
rangeKind: "product",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.VersionExpression,
provenance: rangeProvenance,
primitives: new RangePrimitives(null, null, null, fallbackExtensions)));
return (ranges, normalizedRules);
}
private static Dictionary<string, string> CloneVendorExtensions(IReadOnlyDictionary<string, string> source)
{
var clone = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var pair in source)
{
if (!string.IsNullOrWhiteSpace(pair.Key) && pair.Value is not null)
{
clone[pair.Key] = pair.Value;
}
}
return clone;
}
private sealed record ProductInfo(string? Name, string? VersionExpression);
private async Task<IcsCisaAdvisoryDto> EnrichAdvisoryAsync(IcsCisaAdvisoryDto advisory, CancellationToken cancellationToken)
{

View File

@@ -13,6 +13,7 @@
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
</ItemGroup>
@@ -26,4 +27,4 @@
</AssemblyAttribute>
</ItemGroup>
</Project>
</Project>

View File

@@ -1,4 +1,4 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|FEEDCONN-ICSCISA-02-012 Version range provenance|BE-Conn-ICS-CISA|CONCELIER-LNM-21-001|**TODO (due 2025-10-23)** Promote existing firmware/semver data into `advisory_observations.affected.versions[]` entries with deterministic comparison keys and provenance identifiers (`ics-cisa:{advisoryId}:{product}`). Add regression coverage for mixed firmware strings and raise a Models ticket only when observation schema needs a new comparison helper.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to build observation version entries and log failures without invoking the retired merge helpers.|
|FEEDCONN-ICSCISA-02-012 Version range provenance|BE-Conn-ICS-CISA|CONCELIER-LNM-21-001|**DONE (2025-11-03)** Promote existing firmware/semver data into `advisory_observations.affected.versions[]` entries with deterministic comparison keys and provenance identifiers (`ics-cisa:{advisoryId}:{product}`). Add regression coverage for mixed firmware strings and raise a Models ticket only when observation schema needs a new comparison helper.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to build observation version entries and log failures without invoking the retired merge helpers.<br>2025-11-03: Completed connector now emits semver-aware range rules with provenance, RSS fallback payloads pass the guard, and Fetch/Parse/Map end-to-end coverage succeeds.|

View File

@@ -1,114 +1,831 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Kisa.Internal;
public sealed class KisaDetailParser
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly HtmlContentSanitizer _sanitizer;
public KisaDetailParser(HtmlContentSanitizer sanitizer)
=> _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
public KisaParsedAdvisory Parse(Uri detailApiUri, Uri detailPageUri, byte[] payload)
{
var response = JsonSerializer.Deserialize<KisaDetailResponse>(payload, SerializerOptions)
?? throw new InvalidOperationException("KISA detail payload deserialized to null");
var idx = response.Idx ?? throw new InvalidOperationException("KISA detail missing IDX");
var contentHtml = _sanitizer.Sanitize(response.ContentHtml ?? string.Empty, detailPageUri);
return new KisaParsedAdvisory(
idx,
Normalize(response.Title) ?? idx,
Normalize(response.Summary),
contentHtml,
Normalize(response.Severity),
response.Published,
response.Updated ?? response.Published,
detailApiUri,
detailPageUri,
NormalizeArray(response.CveIds),
MapReferences(response.References),
MapProducts(response.Products));
}
private static IReadOnlyList<string> NormalizeArray(string[]? values)
{
if (values is null || values.Length == 0)
{
return Array.Empty<string>();
}
return values
.Select(Normalize)
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray()!;
}
private static IReadOnlyList<KisaParsedReference> MapReferences(KisaReferenceDto[]? references)
{
if (references is null || references.Length == 0)
{
return Array.Empty<KisaParsedReference>();
}
return references
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
.Select(reference => new KisaParsedReference(reference.Url!, Normalize(reference.Label)))
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<KisaParsedProduct> MapProducts(KisaProductDto[]? products)
{
if (products is null || products.Length == 0)
{
return Array.Empty<KisaParsedProduct>();
}
return products
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
.Select(product => new KisaParsedProduct(
Normalize(product.Vendor),
Normalize(product.Name),
Normalize(product.Versions)))
.ToArray();
}
private static string? Normalize(string? value)
=> string.IsNullOrWhiteSpace(value)
? null
: value.Normalize(NormalizationForm.FormC).Trim();
}
public sealed record KisaParsedAdvisory(
string AdvisoryId,
string Title,
string? Summary,
string ContentHtml,
string? Severity,
DateTimeOffset? Published,
DateTimeOffset? Modified,
Uri DetailApiUri,
Uri DetailPageUri,
IReadOnlyList<string> CveIds,
IReadOnlyList<KisaParsedReference> References,
IReadOnlyList<KisaParsedProduct> Products);
public sealed record KisaParsedReference(string Url, string? Label);
public sealed record KisaParsedProduct(string? Vendor, string? Name, string? Versions);
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.Kisa.Internal;
public sealed class KisaDetailParser
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly Regex CvePattern = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex VendorFromTitlePattern = new(@"\|\s*(?<vendor>[^|]+?)\s+제품", RegexOptions.Compiled);
private readonly HtmlContentSanitizer _sanitizer;
private readonly HtmlParser _htmlParser;
public KisaDetailParser(HtmlContentSanitizer sanitizer)
{
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
_htmlParser = new HtmlParser(new HtmlParserOptions
{
IsKeepingSourceReferences = false,
});
}
public KisaParsedAdvisory Parse(
Uri detailApiUri,
Uri detailPageUri,
byte[] payload,
IReadOnlyDictionary<string, string>? metadata = null)
{
ArgumentNullException.ThrowIfNull(detailApiUri);
ArgumentNullException.ThrowIfNull(detailPageUri);
ArgumentNullException.ThrowIfNull(payload);
if (payload.Length == 0)
{
throw new InvalidOperationException("KISA detail payload was empty.");
}
var parsedJson = TryParseJson(detailApiUri, detailPageUri, payload);
if (parsedJson is not null)
{
return parsedJson;
}
return ParseHtml(detailApiUri, detailPageUri, payload, metadata);
}
private KisaParsedAdvisory? TryParseJson(Uri detailApiUri, Uri detailPageUri, byte[] payload)
{
try
{
var response = JsonSerializer.Deserialize<KisaDetailResponse>(payload, SerializerOptions);
if (response is null || string.IsNullOrWhiteSpace(response.Idx))
{
return null;
}
var contentHtml = _sanitizer.Sanitize(response.ContentHtml ?? string.Empty, detailPageUri);
return new KisaParsedAdvisory(
response.Idx,
Normalize(response.Title) ?? response.Idx!,
Normalize(response.Summary),
contentHtml,
Normalize(response.Severity),
response.Published,
response.Updated ?? response.Published,
detailApiUri,
detailPageUri,
NormalizeArray(response.CveIds),
MapReferences(response.References),
MapProducts(response.Products));
}
catch (JsonException)
{
return null;
}
}
private KisaParsedAdvisory ParseHtml(
Uri detailApiUri,
Uri detailPageUri,
byte[] payload,
IReadOnlyDictionary<string, string>? metadata)
{
var html = DecodePayload(payload);
var document = _htmlParser.ParseDocument(html);
var advisoryId = ResolveIdx(detailApiUri, metadata)
?? throw new InvalidOperationException("KISA detail HTML missing advisory identifier.");
var contentRoot = document.QuerySelector(".domestic_contents") ?? document.Body ?? document.DocumentElement;
var sanitizedContent = _sanitizer.Sanitize(contentRoot?.InnerHtml ?? string.Empty, detailPageUri);
var title = ExtractTitle(document, metadata, advisoryId);
var summary = ExtractSummary(document, sanitizedContent, metadata);
var severity = ExtractSeverity(document);
var published = ExtractPublished(metadata, document);
var modified = ExtractModified(metadata, published);
var cveIds = ExtractCveIds(document);
var references = ExtractHtmlReferences(contentRoot, detailPageUri);
var products = ExtractProducts(document, metadata);
return new KisaParsedAdvisory(
advisoryId,
title,
summary,
sanitizedContent,
severity,
published,
modified,
detailApiUri,
detailPageUri,
cveIds,
references,
products);
}
private static IReadOnlyList<string> NormalizeArray(string[]? values)
{
if (values is null || values.Length == 0)
{
return Array.Empty<string>();
}
return values
.Select(Normalize)
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray()!;
}
private static IReadOnlyList<KisaParsedReference> MapReferences(KisaReferenceDto[]? references)
{
if (references is null || references.Length == 0)
{
return Array.Empty<KisaParsedReference>();
}
return references
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
.Select(reference => new KisaParsedReference(reference.Url!, Normalize(reference.Label)))
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<KisaParsedProduct> MapProducts(KisaProductDto[]? products)
{
if (products is null || products.Length == 0)
{
return Array.Empty<KisaParsedProduct>();
}
return products
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
.Select(product => new KisaParsedProduct(
Normalize(product.Vendor),
Normalize(product.Name),
Normalize(product.Versions)))
.ToArray();
}
private static string DecodePayload(byte[] payload)
=> Encoding.UTF8.GetString(payload);
private static string? ResolveIdx(Uri detailApiUri, IReadOnlyDictionary<string, string>? metadata)
{
if (metadata is not null && metadata.TryGetValue("kisa.idx", out var metadataIdx) && !string.IsNullOrWhiteSpace(metadataIdx))
{
return metadataIdx.Trim();
}
return TryGetQueryValue(detailApiUri, "IDX");
}
private static string ExtractTitle(
IHtmlDocument document,
IReadOnlyDictionary<string, string>? metadata,
string advisoryId)
{
var headerCell = document.QuerySelector("td.bg_tht");
var title = Normalize(headerCell?.TextContent);
var publishedSpan = headerCell?.QuerySelector("span.date");
if (publishedSpan is not null)
{
var dateText = Normalize(publishedSpan.TextContent);
if (!string.IsNullOrEmpty(dateText) && !string.IsNullOrEmpty(title))
{
title = title.Replace(dateText, string.Empty, StringComparison.OrdinalIgnoreCase).Trim();
}
}
if (string.IsNullOrEmpty(title) && metadata is not null && metadata.TryGetValue("kisa.title", out var metaTitle))
{
title = Normalize(metaTitle);
}
return string.IsNullOrEmpty(title) ? advisoryId : title;
}
private string? ExtractSummary(
IHtmlDocument document,
string sanitizedContent,
IReadOnlyDictionary<string, string>? metadata)
{
var overviewParagraph = document.QuerySelectorAll(".domestic_contents p")
.FirstOrDefault(static p => p.TextContent?.Contains("□ 개요", StringComparison.Ordinal) == true);
if (overviewParagraph is not null)
{
foreach (var span in overviewParagraph.QuerySelectorAll("span"))
{
var text = Normalize(span.TextContent);
if (string.IsNullOrEmpty(text))
{
continue;
}
if (text.StartsWith("□", StringComparison.Ordinal))
{
continue;
}
var trimmed = TrimBulletPrefix(text);
if (!string.IsNullOrEmpty(trimmed))
{
return trimmed;
}
}
}
var fallback = ExtractFirstSentence(sanitizedContent);
if (!string.IsNullOrEmpty(fallback))
{
return fallback;
}
if (metadata is not null && metadata.TryGetValue("kisa.title", out var metaTitle))
{
return Normalize(metaTitle);
}
return null;
}
private string? ExtractFirstSentence(string sanitizedContent)
{
if (string.IsNullOrWhiteSpace(sanitizedContent))
{
return null;
}
var fragment = _htmlParser.ParseDocument($"<body>{sanitizedContent}</body>");
if (fragment.Body is null)
{
return null;
}
var firstElement = fragment.Body.Children.FirstOrDefault();
var text = Normalize(firstElement?.TextContent ?? fragment.Body.TextContent);
if (string.IsNullOrEmpty(text))
{
return null;
}
var separatorIndex = text.IndexOfAny(new[] { '。', '.', '!', '?' });
if (separatorIndex > 0 && separatorIndex < text.Length)
{
text = text[..(separatorIndex + 1)].Trim();
}
return TrimBulletPrefix(text);
}
private static string? ExtractSeverity(IHtmlDocument document)
{
foreach (var table in document.QuerySelectorAll("table").OfType<IHtmlTableElement>())
{
if (table.TextContent?.Contains("심각도", StringComparison.OrdinalIgnoreCase) != true)
{
continue;
}
var value = ExtractColumnValue(table, "심각도");
if (!string.IsNullOrWhiteSpace(value))
{
return Normalize(value);
}
}
var labelCell = document.QuerySelectorAll("table td")
.OfType<IHtmlTableCellElement>()
.FirstOrDefault(cell => string.Equals(Normalize(cell.TextContent), "심각도", StringComparison.OrdinalIgnoreCase));
if (labelCell is null)
{
return null;
}
if (labelCell.Closest("table") is not IHtmlTableElement ownerTable)
{
return null;
}
var headerRow = labelCell.ParentElement as IHtmlTableRowElement;
var columnIndex = labelCell.CellIndex;
if (headerRow is null)
{
return null;
}
var rows = ownerTable.Rows.ToArray();
var headerIndex = Array.FindIndex(rows, row => ReferenceEquals(row, headerRow));
if (headerIndex < 0)
{
return null;
}
for (var i = headerIndex + 1; i < rows.Length; i++)
{
var follow = rows[i];
if (follow.Cells.Length <= columnIndex)
{
continue;
}
var value = Normalize(follow.Cells[columnIndex].TextContent);
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
return null;
}
private static DateTimeOffset? ExtractPublished(
IReadOnlyDictionary<string, string>? metadata,
IHtmlDocument document)
{
if (metadata is not null && metadata.TryGetValue("kisa.published", out var publishedText)
&& DateTimeOffset.TryParse(publishedText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published))
{
return published;
}
var dateText = Normalize(document.QuerySelector("td.bg_tht span.date")?.TextContent);
if (string.IsNullOrEmpty(dateText))
{
return null;
}
if (DateTime.TryParseExact(dateText, "yyyy.MM.dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date))
{
return new DateTimeOffset(date, TimeSpan.Zero);
}
return null;
}
private static DateTimeOffset? ExtractModified(
IReadOnlyDictionary<string, string>? metadata,
DateTimeOffset? published)
{
if (metadata is not null && metadata.TryGetValue("kisa.updated", out var updatedText)
&& DateTimeOffset.TryParse(updatedText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var updated))
{
return updated;
}
return published;
}
private static IReadOnlyList<string> ExtractCveIds(IHtmlDocument document)
{
var text = document.Body?.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
return Array.Empty<string>();
}
var matches = CvePattern.Matches(text);
if (matches.Count == 0)
{
return Array.Empty<string>();
}
return matches
.Select(static match => match.Value.ToUpperInvariant())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<KisaParsedReference> ExtractHtmlReferences(IElement? contentRoot, Uri detailPageUri)
{
if (contentRoot is null)
{
return Array.Empty<KisaParsedReference>();
}
var anchors = contentRoot.QuerySelectorAll("a[href]");
if (anchors.Length == 0)
{
return Array.Empty<KisaParsedReference>();
}
var references = new List<KisaParsedReference>(anchors.Length);
foreach (var anchor in anchors)
{
var href = anchor.GetAttribute("href");
if (string.IsNullOrWhiteSpace(href))
{
continue;
}
if (!Uri.TryCreate(detailPageUri, href, out var normalized))
{
continue;
}
if (!string.Equals(normalized.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
&& !string.Equals(normalized.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var label = Normalize(anchor.TextContent);
references.Add(new KisaParsedReference(normalized.ToString(), label));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<KisaParsedProduct> ExtractProducts(
IHtmlDocument document,
IReadOnlyDictionary<string, string>? metadata)
{
var root = document.QuerySelector(".domestic_contents") ?? document.Body ?? document.DocumentElement;
if (root is null)
{
return Array.Empty<KisaParsedProduct>();
}
var table = FindProductTable(root);
if (table is null || table.Rows.Length <= 1)
{
return Array.Empty<KisaParsedProduct>();
}
var defaultVendor = ExtractVendorHint(document, metadata);
var accumulators = new List<ProductAccumulator>();
var lookup = new Dictionary<string, ProductAccumulator>(StringComparer.OrdinalIgnoreCase);
string? currentProduct = null;
for (var i = 1; i < table.Rows.Length; i++)
{
var row = table.Rows[i];
if (row.Cells.Length == 0)
{
continue;
}
string? productName = null;
string? affected = null;
if (row.Cells.Length >= 3)
{
productName = Normalize(row.Cells[0].TextContent);
affected = Normalize(row.Cells[1].TextContent);
}
else
{
affected = Normalize(row.Cells[0].TextContent);
}
if (!string.IsNullOrEmpty(productName))
{
currentProduct = productName;
}
if (string.IsNullOrEmpty(currentProduct))
{
continue;
}
if (!lookup.TryGetValue(currentProduct, out var accumulator))
{
accumulator = new ProductAccumulator(currentProduct);
lookup.Add(currentProduct, accumulator);
accumulators.Add(accumulator);
}
if (!string.IsNullOrEmpty(affected))
{
accumulator.Impacted.Add(affected);
}
}
if (accumulators.Count == 0)
{
return Array.Empty<KisaParsedProduct>();
}
var products = new List<KisaParsedProduct>(accumulators.Count);
foreach (var accumulator in accumulators)
{
var (vendor, name) = SplitVendorAndName(accumulator.RawName, defaultVendor);
var versions = ComposeVersionString(accumulator.Impacted);
products.Add(new KisaParsedProduct(vendor, name, versions));
}
return products;
}
private static IHtmlTableElement? FindProductTable(IElement root)
{
var tables = root.QuerySelectorAll("table");
foreach (var element in tables.OfType<IHtmlTableElement>())
{
var header = element.Rows.FirstOrDefault();
if (header is null)
{
continue;
}
foreach (var cell in header.Cells)
{
var text = Normalize(cell.TextContent);
if (!string.IsNullOrEmpty(text)
&& text.Contains("영향받는 버전", StringComparison.OrdinalIgnoreCase))
{
return element;
}
}
}
return null;
}
private static string? ExtractVendorHint(
IHtmlDocument document,
IReadOnlyDictionary<string, string>? metadata)
{
var headerCell = document.QuerySelector("td.bg_tht");
var headerText = Normalize(headerCell?.TextContent);
if (!string.IsNullOrEmpty(headerText))
{
var match = VendorFromTitlePattern.Match(headerText);
if (match.Success)
{
return Normalize(match.Groups["vendor"].Value);
}
}
if (metadata is not null && metadata.TryGetValue("kisa.title", out var metaTitle))
{
var normalized = Normalize(metaTitle);
if (!string.IsNullOrEmpty(normalized))
{
var match = VendorFromTitlePattern.Match(normalized);
if (match.Success)
{
return Normalize(match.Groups["vendor"].Value);
}
}
}
return null;
}
private static string? ExtractColumnValue(IHtmlTableElement table, string headerLabel)
{
if (table.Rows.Length < 2)
{
return null;
}
var rows = table.Rows;
for (var rowIndex = 0; rowIndex < rows.Length; rowIndex++)
{
var row = rows[rowIndex];
for (var columnIndex = 0; columnIndex < row.Cells.Length; columnIndex++)
{
var headerText = Normalize(row.Cells[columnIndex].TextContent);
if (string.IsNullOrEmpty(headerText)
|| !headerText.Contains(headerLabel, StringComparison.OrdinalIgnoreCase))
{
continue;
}
for (var nextRowIndex = rowIndex + 1; nextRowIndex < rows.Length; nextRowIndex++)
{
var candidateRow = rows[nextRowIndex];
if (candidateRow.Cells.Length <= columnIndex)
{
continue;
}
var value = Normalize(candidateRow.Cells[columnIndex].TextContent);
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
}
return null;
}
private static (string? Vendor, string? Name) SplitVendorAndName(string rawName, string? defaultVendor)
{
var normalized = Normalize(rawName);
if (string.IsNullOrEmpty(normalized))
{
return (defaultVendor, null);
}
var tokens = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (tokens.Length <= 1)
{
return (defaultVendor ?? normalized, normalized);
}
var englishVendor = tokens[0];
var name = normalized[(englishVendor.Length)..].Trim();
if (string.IsNullOrEmpty(name))
{
name = normalized;
}
var vendor = defaultVendor ?? englishVendor;
return (vendor, name);
}
private static string? ComposeVersionString(IEnumerable<string> impacted)
{
var normalized = impacted
.Select(Normalize)
.Where(static value => !string.IsNullOrEmpty(value))
.Select(static value => value!)
.ToList();
if (normalized.Count == 0)
{
return null;
}
if (normalized.Count == 1)
{
return normalized[0];
}
if (normalized.Any(ContainsRangeMarker))
{
return normalized[0];
}
var prefix = FindCommonPrefix(normalized);
if (!string.IsNullOrEmpty(prefix))
{
var suffix = normalized[^1][prefix.Length..].TrimStart();
if (!string.IsNullOrEmpty(suffix))
{
return $"{normalized[0]} ~ {suffix}";
}
}
return $"{normalized[0]} ~ {normalized[^1]}";
}
private static string FindCommonPrefix(IReadOnlyList<string> values)
{
if (values.Count == 0)
{
return string.Empty;
}
var prefix = values[0];
for (var i = 1; i < values.Count && prefix.Length > 0; i++)
{
var candidate = values[i];
var max = Math.Min(prefix.Length, candidate.Length);
var index = 0;
while (index < max && prefix[index] == candidate[index])
{
index++;
}
prefix = prefix[..index];
}
if (prefix.Length == 0)
{
return string.Empty;
}
var lastSpace = prefix.LastIndexOf(' ');
if (lastSpace < 0)
{
return string.Empty;
}
return prefix[..(lastSpace + 1)];
}
private static bool ContainsRangeMarker(string value)
=> value.Contains('~', StringComparison.Ordinal)
|| value.Contains("이상", StringComparison.Ordinal)
|| value.Contains("이하", StringComparison.Ordinal)
|| value.Contains("초과", StringComparison.Ordinal)
|| value.Contains("미만", StringComparison.Ordinal);
private static string? Normalize(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var normalized = value.Normalize(NormalizationForm.FormC).Trim();
var builder = new StringBuilder(normalized.Length);
var previousWhitespace = false;
foreach (var ch in normalized)
{
if (char.IsWhiteSpace(ch))
{
if (!previousWhitespace)
{
builder.Append(' ');
previousWhitespace = true;
}
}
else
{
builder.Append(ch);
previousWhitespace = false;
}
}
return builder.ToString().Trim();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart();
while (trimmed.Length > 0 && (trimmed[0] is 'o' or '•' or '-' or 'ㆍ'))
{
trimmed = trimmed[1..].TrimStart();
}
return trimmed.Trim();
}
private static string? TryGetQueryValue(Uri uri, string key)
{
if (string.IsNullOrEmpty(uri.Query))
{
return null;
}
foreach (var pair in uri.Query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=', StringComparison.Ordinal);
if (separatorIndex <= 0)
{
continue;
}
var candidateKey = pair[..separatorIndex];
if (!candidateKey.Equals(key, StringComparison.OrdinalIgnoreCase))
{
continue;
}
return Uri.UnescapeDataString(pair[(separatorIndex + 1)..]);
}
return null;
}
private sealed class ProductAccumulator
{
public ProductAccumulator(string rawName)
{
RawName = rawName;
}
public string RawName { get; }
public List<string> Impacted { get; } = new();
}
}
public sealed record KisaParsedAdvisory(
string AdvisoryId,
string Title,
string? Summary,
string ContentHtml,
string? Severity,
DateTimeOffset? Published,
DateTimeOffset? Modified,
Uri DetailApiUri,
Uri DetailPageUri,
IReadOnlyList<string> CveIds,
IReadOnlyList<KisaParsedReference> References,
IReadOnlyList<KisaParsedProduct> Products);
public sealed record KisaParsedReference(string Url, string? Label);
public sealed record KisaParsedProduct(string? Vendor, string? Name, string? Versions);

View File

@@ -6,13 +6,14 @@ namespace StellaOps.Concelier.Connector.Kisa.Internal;
internal static class KisaDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(KisaFeedItem item)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["kisa.idx"] = item.AdvisoryId,
["kisa.detailPage"] = item.DetailPageUri.ToString(),
["kisa.published"] = item.Published.ToString("O"),
};
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["kisa.idx"] = item.AdvisoryId,
["kisa.detailApi"] = item.DetailApiUri.ToString(),
["kisa.detailPage"] = item.DetailPageUri.ToString(),
["kisa.published"] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Title))
{

View File

@@ -1,11 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.Kisa.Internal;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.Kisa.Internal;
internal static class KisaMapper
{
public static Advisory Map(KisaParsedAdvisory dto, DocumentRecord document, DateTimeOffset recordedAt)
@@ -96,50 +97,410 @@ internal static class KisaMapper
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var vendor = string.IsNullOrWhiteSpace(product.Vendor) ? "Unknown" : product.Vendor!;
var name = product.Name;
var identifier = string.IsNullOrWhiteSpace(name) ? vendor : $"{vendor} {name}";
var provenance = new AdvisoryProvenance(
KisaConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var versionRanges = string.IsNullOrWhiteSpace(product.Versions)
? Array.Empty<AffectedVersionRange>()
: new[]
{
new AffectedVersionRange(
rangeKind: "string",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.Versions,
provenance: new AdvisoryProvenance(
KisaConnectorPlugin.SourceName,
"package-range",
product.Versions,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges }))
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: versionRanges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
}
foreach (var product in dto.Products)
{
var vendor = string.IsNullOrWhiteSpace(product.Vendor) ? "Unknown" : product.Vendor!;
var name = product.Name;
var identifier = string.IsNullOrWhiteSpace(name) ? vendor : $"{vendor} {name}";
var normalizedIdentifier = CreateSlug(identifier);
var rangeProvenanceKey = $"kisa:{dto.AdvisoryId}:{normalizedIdentifier}";
var artifacts = BuildVersionArtifacts(product, rangeProvenanceKey, recordedAt);
var fieldMasks = new HashSet<string>(StringComparer.Ordinal)
{
ProvenanceFieldMasks.AffectedPackages
};
if (artifacts.Ranges.Count > 0)
{
fieldMasks.Add(ProvenanceFieldMasks.VersionRanges);
}
if (artifacts.NormalizedVersions.Count > 0)
{
fieldMasks.Add(ProvenanceFieldMasks.NormalizedVersions);
}
var packageProvenance = new AdvisoryProvenance(
KisaConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
fieldMasks);
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: artifacts.Ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { packageProvenance },
normalizedVersions: artifacts.NormalizedVersions));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> NormalizedVersions) BuildVersionArtifacts(
KisaParsedProduct product,
string provenanceValue,
DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(product.Versions))
{
var fallback = CreateFallbackRange(product.Versions ?? string.Empty, provenanceValue, recordedAt);
return (new[] { fallback }, Array.Empty<NormalizedVersionRule>());
}
var segment = product.Versions.Trim();
var result = ParseRangeSegment(segment, provenanceValue, recordedAt);
var ranges = new[] { result.Range };
var normalized = result.NormalizedRule is null
? Array.Empty<NormalizedVersionRule>()
: new[] { result.NormalizedRule };
return (ranges, normalized);
}
private static (AffectedVersionRange Range, NormalizedVersionRule? NormalizedRule) ParseRangeSegment(
string segment,
string provenanceValue,
DateTimeOffset recordedAt)
{
var trimmed = segment.Trim();
if (trimmed.Length == 0)
{
return (CreateFallbackRange(segment, provenanceValue, recordedAt), null);
}
var matches = VersionPattern.Matches(trimmed);
if (matches.Count == 0)
{
return (CreateFallbackRange(segment, provenanceValue, recordedAt), null);
}
var startMatch = matches[0];
var startVersion = startMatch.Value;
string? endVersion = matches.Count > 1 ? matches[1].Value : null;
var prefix = trimmed[..startMatch.Index].Trim();
var startContext = ExtractSpan(trimmed, startMatch.Index + startMatch.Length, endVersion is not null ? matches[1].Index : trimmed.Length).Trim();
var endContext = endVersion is not null
? trimmed[(matches[1].Index + matches[1].Length)..].Trim()
: string.Empty;
var introducedInclusive = DetermineStartInclusivity(prefix, startContext, trimmed);
var endContextForInclusivity = endVersion is not null ? endContext : startContext;
var fixedInclusive = DetermineEndInclusivity(endContextForInclusivity, trimmed);
var hasInclusiveLowerMarker = ContainsAny(prefix, InclusiveStartMarkers) || ContainsAny(startContext, InclusiveStartMarkers);
var hasExclusiveLowerMarker = ContainsAny(prefix, ExclusiveStartMarkers) || ContainsAny(startContext, ExclusiveStartMarkers);
var hasInclusiveUpperMarker = ContainsAny(startContext, InclusiveEndMarkers) || ContainsAny(endContext, InclusiveEndMarkers);
var hasExclusiveUpperMarker = ContainsAny(startContext, ExclusiveEndMarkers) || ContainsAny(endContext, ExclusiveEndMarkers);
var hasUpperMarker = hasInclusiveUpperMarker || hasExclusiveUpperMarker;
var hasLowerMarker = hasInclusiveLowerMarker || hasExclusiveLowerMarker;
var introducedNormalized = TryFormatSemVer(startVersion);
var fixedNormalized = endVersion is not null ? TryFormatSemVer(endVersion) : null;
if (introducedNormalized is null || (endVersion is not null && fixedNormalized is null))
{
return (CreateFallbackRange(segment, provenanceValue, recordedAt), null);
}
var coercedUpperOnly = endVersion is null && hasUpperMarker && !hasLowerMarker;
if (coercedUpperOnly)
{
fixedNormalized = introducedNormalized;
introducedNormalized = null;
fixedInclusive = hasInclusiveUpperMarker && !hasExclusiveUpperMarker;
}
var constraintExpression = BuildConstraintExpression(
introducedNormalized,
introducedInclusive,
fixedNormalized,
fixedInclusive);
var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["kisa.range.raw"] = trimmed,
["kisa.version.start.raw"] = startVersion
};
if (introducedNormalized is not null)
{
vendorExtensions["kisa.version.start.normalized"] = introducedNormalized;
}
if (!string.IsNullOrWhiteSpace(prefix))
{
vendorExtensions["kisa.range.prefix"] = prefix;
}
if (coercedUpperOnly)
{
vendorExtensions["kisa.version.end.raw"] = startVersion;
vendorExtensions["kisa.version.end.normalized"] = fixedNormalized!;
}
if (endVersion is not null)
{
vendorExtensions["kisa.version.end.raw"] = endVersion;
vendorExtensions["kisa.version.end.normalized"] = fixedNormalized!;
}
if (!string.IsNullOrWhiteSpace(startContext))
{
vendorExtensions["kisa.range.start.context"] = startContext;
}
if (!string.IsNullOrWhiteSpace(endContext))
{
vendorExtensions["kisa.range.end.context"] = endContext;
}
if (!string.IsNullOrWhiteSpace(constraintExpression))
{
vendorExtensions["kisa.range.normalized"] = constraintExpression!;
}
var semVerPrimitive = new SemVerPrimitive(
Introduced: introducedNormalized,
IntroducedInclusive: introducedInclusive,
Fixed: fixedNormalized,
FixedInclusive: fixedInclusive,
LastAffected: fixedNormalized,
LastAffectedInclusive: fixedNormalized is not null ? fixedInclusive : introducedInclusive,
ConstraintExpression: constraintExpression,
ExactValue: fixedNormalized is null && string.IsNullOrWhiteSpace(constraintExpression) ? introducedNormalized : null);
var range = new AffectedVersionRange(
rangeKind: "product",
introducedVersion: semVerPrimitive.Introduced,
fixedVersion: semVerPrimitive.Fixed,
lastAffectedVersion: semVerPrimitive.LastAffected,
rangeExpression: trimmed,
provenance: new AdvisoryProvenance(
KisaConnectorPlugin.SourceName,
"package-range",
provenanceValue,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges }),
primitives: new RangePrimitives(semVerPrimitive, null, null, vendorExtensions));
var normalizedRule = semVerPrimitive.ToNormalizedVersionRule(provenanceValue);
return (range, normalizedRule);
}
private static AffectedVersionRange CreateFallbackRange(string raw, string provenanceValue, DateTimeOffset recordedAt)
{
var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(raw))
{
vendorExtensions["kisa.range.raw"] = raw.Trim();
}
return new AffectedVersionRange(
rangeKind: "string",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: raw,
provenance: new AdvisoryProvenance(
KisaConnectorPlugin.SourceName,
"package-range",
provenanceValue,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges }),
primitives: new RangePrimitives(null, null, null, vendorExtensions));
}
private static string ExtractSpan(string source, int start, int end)
{
if (start >= end || start >= source.Length)
{
return string.Empty;
}
end = Math.Min(end, source.Length);
return source[start..end];
}
private static string? TryFormatSemVer(string version)
{
var segments = version.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (segments.Length == 0)
{
return null;
}
if (!TryParseInt(segments[0], out var major))
{
return null;
}
var minor = segments.Length > 1 && TryParseInt(segments[1], out var minorValue) ? minorValue : 0;
var patch = segments.Length > 2 && TryParseInt(segments[2], out var patchValue) ? patchValue : 0;
var baseVersion = $"{major}.{minor}.{patch}";
if (segments.Length <= 3)
{
return baseVersion;
}
var extraIdentifiers = segments
.Skip(3)
.Select(TrimLeadingZeros)
.Where(static part => part.Length > 0)
.ToArray();
if (extraIdentifiers.Length == 0)
{
extraIdentifiers = new[] { "0" };
}
var allIdentifiers = new[] { "fw" }.Concat(extraIdentifiers);
return $"{baseVersion}-{string.Join('.', allIdentifiers)}";
}
private static string TrimLeadingZeros(string value)
{
var trimmed = value.TrimStart('0');
return trimmed.Length == 0 ? "0" : trimmed;
}
private static bool TryParseInt(string value, out int result)
=> int.TryParse(value.Trim(), out result);
private static bool DetermineStartInclusivity(string prefix, string context, string fullSegment)
{
if (ContainsAny(prefix, ExclusiveStartMarkers) || ContainsAny(context, ExclusiveStartMarkers))
{
return false;
}
if (fullSegment.Contains('~', StringComparison.Ordinal))
{
return true;
}
if (ContainsAny(prefix, InclusiveStartMarkers) || ContainsAny(context, InclusiveStartMarkers))
{
return true;
}
return true;
}
private static bool DetermineEndInclusivity(string context, string fullSegment)
{
if (string.IsNullOrWhiteSpace(context))
{
return true;
}
if (ContainsAny(context, ExclusiveEndMarkers))
{
return false;
}
if (fullSegment.Contains('~', StringComparison.Ordinal))
{
return true;
}
if (ContainsAny(context, InclusiveEndMarkers))
{
return true;
}
return true;
}
private static string? BuildConstraintExpression(
string? introduced,
bool introducedInclusive,
string? fixedVersion,
bool fixedInclusive)
{
var segments = new List<string>(capacity: 2);
if (!string.IsNullOrWhiteSpace(introduced))
{
segments.Add($"{(introducedInclusive ? ">=" : ">")} {introduced}");
}
if (!string.IsNullOrWhiteSpace(fixedVersion))
{
segments.Add($"{(fixedInclusive ? "<=" : "<")} {fixedVersion}");
}
return segments.Count == 0 ? null : string.Join(" ", segments);
}
private static bool ContainsAny(string? value, IReadOnlyCollection<string> markers)
{
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
foreach (var marker in markers)
{
if (value.Contains(marker, StringComparison.Ordinal))
{
return true;
}
}
return false;
}
private static string CreateSlug(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "kisa-product";
}
Span<char> buffer = stackalloc char[value.Length];
var index = 0;
foreach (var ch in value.ToLowerInvariant())
{
if (char.IsLetterOrDigit(ch))
{
buffer[index++] = ch;
}
else if (char.IsWhiteSpace(ch) || ch is '-' or '_' or '.' or '/')
{
if (index == 0 || buffer[index - 1] == '-')
{
continue;
}
buffer[index++] = '-';
}
}
if (index == 0)
{
return "kisa-product";
}
var slug = new string(buffer[..index]).Trim('-');
return string.IsNullOrWhiteSpace(slug) ? "kisa-product" : slug;
}
private static readonly Regex VersionPattern = new(@"\d+(?:\.\d+){1,3}", RegexOptions.Compiled);
private static readonly string[] InclusiveStartMarkers = { "이상" };
private static readonly string[] ExclusiveStartMarkers = { "초과" };
private static readonly string[] InclusiveEndMarkers = { "이하" };
private static readonly string[] ExclusiveEndMarkers = { "미만" };
}

View File

@@ -131,17 +131,24 @@ public sealed class KisaConnector : IFeedConnector
var category = item.Category;
_diagnostics.DetailAttempt(category);
try
{
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailApiUri.ToString(), cancellationToken).ConfigureAwait(false);
var request = new SourceFetchRequest(KisaOptions.HttpClientName, SourceName, item.DetailApiUri)
{
Metadata = KisaDocumentMetadata.CreateMetadata(item),
AcceptHeaders = new[] { "application/json", "text/json" },
ETag = existing?.Etag,
LastModified = existing?.LastModified,
TimeoutOverride = _options.RequestTimeout,
};
try
{
var detailUri = item.DetailPageUri;
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri.ToString(), cancellationToken).ConfigureAwait(false);
var request = new SourceFetchRequest(KisaOptions.HttpClientName, SourceName, detailUri)
{
Metadata = KisaDocumentMetadata.CreateMetadata(item),
AcceptHeaders = new[]
{
"text/html",
"application/xhtml+xml",
"application/json",
"text/json",
},
ETag = existing?.Etag,
LastModified = existing?.LastModified,
TimeoutOverride = _options.RequestTimeout,
};
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
if (result.IsNotModified)
@@ -261,19 +268,17 @@ public sealed class KisaConnector : IFeedConnector
throw;
}
KisaParsedAdvisory parsed;
try
{
var apiUri = new Uri(document.Uri);
var pageUri = document.Metadata is not null && document.Metadata.TryGetValue("kisa.detailPage", out var pageValue)
? new Uri(pageValue)
: apiUri;
parsed = _detailParser.Parse(apiUri, pageUri, payload);
}
catch (Exception ex)
{
_diagnostics.ParseFailure(category, "parse");
_logger.LogError(ex, "KISA failed to parse detail {DocumentId}", document.Id);
KisaParsedAdvisory parsed;
try
{
var apiUri = TryGetUri(document.Metadata, "kisa.detailApi") ?? new Uri(document.Uri);
var pageUri = TryGetUri(document.Metadata, "kisa.detailPage") ?? new Uri(document.Uri);
parsed = _detailParser.Parse(apiUri, pageUri, payload, document.Metadata);
}
catch (Exception ex)
{
_diagnostics.ParseFailure(category, "parse");
_logger.LogError(ex, "KISA failed to parse detail {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
@@ -296,8 +301,23 @@ public sealed class KisaConnector : IFeedConnector
.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
}
private static Uri? TryGetUri(IReadOnlyDictionary<string, string>? metadata, string key)
{
if (metadata is null)
{
return null;
}
if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null;
}
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);

View File

@@ -1,4 +1,4 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|FEEDCONN-KISA-02-008 Firmware range provenance|BE-Conn-KISA, Models|CONCELIER-LNM-21-001|**TODO (due 2025-10-24)** Define comparison helpers for Hangul-labelled firmware ranges (`XFU 1.0.1.0084 ~ 2.0.1.0034`) and map them into `advisory_observations.affected.versions[]` with provenance tags. Coordinate with Models only if a new comparison scheme is required, then update localisation notes and fixtures for the Link-Not-Merge schema.|
|FEEDCONN-KISA-02-008 Firmware range provenance|BE-Conn-KISA, Models|CONCELIER-LNM-21-001|**DONE (2025-11-04)** Defined comparison helpers for Hangul-labelled firmware ranges (`XFU 1.0.1.0084 ~ 2.0.1.0034`) and mapped them into `advisory_observations.affected.versions[]` with provenance tags. Coordinated localisation notes/fixtures for Link-Not-Merge schema.<br>2025-11-03: Kicking off range normalization + provenance mapping; auditing existing mapper/tests before implementing semver/firmware helper.<br>2025-11-03: Implemented SemVer normalization pipeline with provenance slugs, added vendor extension masks, and refreshed end-to-end tests to cover normalized rules; Continue reviewing additional range phrasings (`미만`/`초과`) before marking DONE.<br>2025-11-03: Added coverage for exclusive/inclusive single-ended ranges and fallback handling (`미만`, `이하`, `초과`, non-numeric text); mapper now emits deterministic SemVer primitives and normalized rules for those phrasings—final pass pending broader fixture sweep.<br>2025-11-03: Switched detail fetch to HTML (`detailDos.do`) and introduced DOM-based parser + fixtures so advisory products/ranges persist even when the JSON detail API rejects unauthenticated clients.<br>2025-11-04: Parser severity/table extraction tightened and dedicated HTML fixture-powered tests ensure normalized ranges, vendor extensions, and severity survive the DOM path; integration suite runs against HTML snapshots.|

View File

@@ -506,19 +506,23 @@ public static class SemVerRangeRuleBuilder
}
var candidate = RemoveLeadingV(trimmed);
if (SemanticVersion.TryParse(candidate, out var semanticVersion))
{
normalized = FormatVersion(semanticVersion);
return true;
}
if (trimmed.IndexOfAny(new[] { '*', 'x', 'X' }) >= 0)
{
return false;
}
normalized = candidate;
return true;
if (!SemanticVersion.TryParse(candidate, out var semanticVersion))
{
var expanded = ExpandSemanticVersion(candidate);
if (!SemanticVersion.TryParse(expanded, out semanticVersion))
{
if (trimmed.IndexOfAny(new[] { '*', 'x', 'X' }) >= 0)
{
return false;
}
normalized = candidate;
return true;
}
}
normalized = FormatVersion(semanticVersion);
return true;
}
private static bool TryParseSemanticVersion(string value, [NotNullWhen(true)] out SemanticVersion version, out string normalized)

View File

@@ -13,13 +13,15 @@ using StellaOps.Concelier.Storage.Mongo.Aliases;
namespace StellaOps.Concelier.Storage.Mongo.Advisories;
public sealed class AdvisoryStore : IAdvisoryStore
{
public sealed class AdvisoryStore : IAdvisoryStore
{
private readonly IMongoDatabase _database;
private readonly IMongoCollection<AdvisoryDocument> _collection;
private readonly ILogger<AdvisoryStore> _logger;
private readonly IAliasStore _aliasStore;
private readonly TimeProvider _timeProvider;
private readonly MongoStorageOptions _options;
private IMongoCollection<AdvisoryDocument>? _legacyCollection;
public AdvisoryStore(
IMongoDatabase database,
@@ -28,8 +30,8 @@ public sealed class AdvisoryStore : IAdvisoryStore
IOptions<MongoStorageOptions> options,
TimeProvider? timeProvider = null)
{
_collection = (database ?? throw new ArgumentNullException(nameof(database)))
.GetCollection<AdvisoryDocument>(MongoStorageDefaults.Collections.Advisory);
_database = database ?? throw new ArgumentNullException(nameof(database));
_collection = _database.GetCollection<AdvisoryDocument>(MongoStorageDefaults.Collections.Advisory);
_aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
@@ -69,14 +71,7 @@ public sealed class AdvisoryStore : IAdvisoryStore
var options = new ReplaceOptions { IsUpsert = true };
var filter = Builders<AdvisoryDocument>.Filter.Eq(x => x.AdvisoryKey, advisory.AdvisoryKey);
if (session is null)
{
await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
else
{
await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false);
}
await ReplaceAsync(filter, document, options, session, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey);
var aliasEntries = BuildAliasEntries(advisory);
@@ -129,6 +124,71 @@ public sealed class AdvisoryStore : IAdvisoryStore
return cursor.Select(static doc => Deserialize(doc.Payload)).ToArray();
}
private async Task ReplaceAsync(
FilterDefinition<AdvisoryDocument> filter,
AdvisoryDocument document,
ReplaceOptions options,
IClientSessionHandle? session,
CancellationToken cancellationToken)
{
try
{
if (session is null)
{
await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
else
{
await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false);
}
}
catch (MongoWriteException ex) when (IsNamespaceViewError(ex))
{
var legacyCollection = await GetLegacyAdvisoryCollectionAsync(cancellationToken).ConfigureAwait(false);
if (session is null)
{
await legacyCollection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
else
{
await legacyCollection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false);
}
}
}
private static bool IsNamespaceViewError(MongoWriteException ex)
=> ex?.WriteError?.Code == 166 ||
(ex?.WriteError?.Message?.Contains("is a view", StringComparison.OrdinalIgnoreCase) ?? false);
private async ValueTask<IMongoCollection<AdvisoryDocument>> GetLegacyAdvisoryCollectionAsync(CancellationToken cancellationToken)
{
if (_legacyCollection is not null)
{
return _legacyCollection;
}
var filter = new BsonDocument("name", MongoStorageDefaults.Collections.Advisory);
using var cursor = await _database
.ListCollectionsAsync(new ListCollectionsOptions { Filter = filter }, cancellationToken)
.ConfigureAwait(false);
var info = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("Advisory collection metadata not found.");
if (!info.TryGetValue("options", out var optionsValue) || optionsValue is not BsonDocument optionsDocument)
{
throw new InvalidOperationException("Advisory view options missing.");
}
if (!optionsDocument.TryGetValue("viewOn", out var viewOnValue) || viewOnValue.BsonType != BsonType.String)
{
throw new InvalidOperationException("Advisory view target not specified.");
}
var targetName = viewOnValue.AsString;
_legacyCollection = _database.GetCollection<AdvisoryDocument>(targetName);
return _legacyCollection;
}
public async IAsyncEnumerable<Advisory> StreamAsync([EnumeratorCancellation] CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
var options = new FindOptions<AdvisoryDocument>

View File

@@ -46,14 +46,42 @@ public sealed class AliasStore : IAliasStore
});
}
if (documents.Count > 0)
{
await _collection.InsertManyAsync(
documents,
new InsertManyOptions { IsOrdered = false },
cancellationToken).ConfigureAwait(false);
}
}
if (documents.Count > 0)
{
try
{
await _collection.InsertManyAsync(
documents,
new InsertManyOptions { IsOrdered = false },
cancellationToken).ConfigureAwait(false);
}
catch (MongoBulkWriteException<AliasDocument> ex) when (ex.WriteErrors.Any(error => error.Category == ServerErrorCategory.DuplicateKey))
{
foreach (var writeError in ex.WriteErrors.Where(error => error.Category == ServerErrorCategory.DuplicateKey))
{
var duplicateDocument = documents.ElementAtOrDefault(writeError.Index);
_logger.LogError(
ex,
"Alias duplicate detected while inserting {Scheme}:{Value} for advisory {AdvisoryKey}. Existing aliases: {Existing}",
duplicateDocument?.Scheme,
duplicateDocument?.Value,
duplicateDocument?.AdvisoryKey,
string.Join(", ", aliasList.Select(a => $"{a.Scheme}:{a.Value}")));
}
throw;
}
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
{
_logger.LogError(
ex,
"Alias duplicate detected while inserting aliases for advisory {AdvisoryKey}. Aliases: {Aliases}",
advisoryKey,
string.Join(", ", aliasList.Select(a => $"{a.Scheme}:{a.Value}")));
throw;
}
}
}
if (aliasList.Length == 0)
{

View File

@@ -70,11 +70,11 @@ public class IcsCisaConnectorMappingTests
}
[Fact]
public void BuildAffectedPackages_EmitsProductRangesWithSemVer()
{
var dto = new IcsCisaAdvisoryDto
{
AdvisoryId = "ICSA-25-456-02",
public void BuildAffectedPackages_EmitsProductRangesWithSemVer()
{
var dto = new IcsCisaAdvisoryDto
{
AdvisoryId = "ICSA-25-456-02",
Title = "Vendor Advisory",
Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-456-02",
DescriptionHtml = "",
@@ -89,13 +89,54 @@ public class IcsCisaConnectorMappingTests
var productPackage = Assert.Single(packages);
Assert.Equal(AffectedPackageTypes.IcsVendor, productPackage.Type);
Assert.Equal("ControlSuite", productPackage.Identifier);
var range = Assert.Single(productPackage.VersionRanges);
Assert.Equal("product", range.RangeKind);
Assert.Equal("4.2", range.RangeExpression);
Assert.NotNull(range.Primitives);
Assert.Equal("Example Corp", range.Primitives!.VendorExtensions!["ics.vendors"]);
Assert.Equal("ControlSuite", range.Primitives.VendorExtensions!["ics.product"]);
Assert.NotNull(range.Primitives.SemVer);
Assert.Equal("4.2.0", range.Primitives.SemVer!.ExactValue);
}
}
var range = Assert.Single(productPackage.VersionRanges);
Assert.Equal("product", range.RangeKind);
Assert.Equal("4.2.0", range.RangeExpression);
Assert.NotNull(range.Primitives);
Assert.Equal("Example Corp", range.Primitives!.VendorExtensions!["ics.vendors"]);
Assert.Equal("ControlSuite", range.Primitives.VendorExtensions!["ics.product"]);
Assert.True(range.Primitives.VendorExtensions!.ContainsKey("ics.range.expression"));
Assert.NotNull(range.Primitives.SemVer);
Assert.Equal("4.2.0", range.Primitives.SemVer!.ExactValue);
Assert.Equal("ics-cisa:ICSA-25-456-02:controlsuite", range.Provenance.Value);
var normalizedRule = Assert.Single(productPackage.NormalizedVersions);
Assert.Equal("semver", normalizedRule.Scheme);
Assert.Equal("exact", normalizedRule.Type);
Assert.Equal("4.2.0", normalizedRule.Value);
Assert.Equal("ics-cisa:ICSA-25-456-02:controlsuite", normalizedRule.Notes);
var packageProvenance = Assert.Single(productPackage.Provenance);
Assert.Contains(ProvenanceFieldMasks.AffectedPackages, packageProvenance.FieldMask);
Assert.Contains(ProvenanceFieldMasks.VersionRanges, packageProvenance.FieldMask);
Assert.Contains(ProvenanceFieldMasks.NormalizedVersions, packageProvenance.FieldMask);
}
[Fact]
public void BuildAffectedPackages_NormalizesRangeExpressions()
{
var dto = new IcsCisaAdvisoryDto
{
AdvisoryId = "ICSA-25-789-03",
Title = "Range Advisory",
Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-789-03",
DescriptionHtml = "",
Published = RecordedAt,
Vendors = new[] { "Range Corp" },
Products = new[] { "Control Suite Firmware 1.0 - 2.0" }
};
var packages = IcsCisaConnector.BuildAffectedPackages(dto, RecordedAt);
var productPackage = Assert.Single(packages);
Assert.Equal("Control Suite Firmware", productPackage.Identifier);
var range = Assert.Single(productPackage.VersionRanges);
Assert.Equal("1.0.0 - 2.0.0", range.RangeExpression);
Assert.NotNull(range.Primitives);
Assert.Equal("ics-cisa:ICSA-25-789-03:control-suite-firmware", range.Provenance.Value);
var rule = Assert.Single(productPackage.NormalizedVersions);
Assert.Equal("semver", rule.Scheme);
Assert.Equal("range", rule.Type);
Assert.Equal("1.0.0", rule.Min);
Assert.Equal("2.0.0", rule.Max);
Assert.Equal("ics-cisa:ICSA-25-789-03:control-suite-firmware", rule.Notes);
}
}

View File

@@ -50,8 +50,7 @@ public sealed class IcsCisaConnectorTests : IAsyncLifetime
Assert.Equal(2, advisories.Count);
var icsa = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "ICSA-25-123-01");
Console.WriteLine("ProductsRaw:" + string.Join("|", icsa.AffectedPackages.SelectMany(p => p.Provenance).Select(p => p.Value ?? "<null>")));
var icsa = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "ICSA-25-123-01");
Assert.Contains("CVE-2024-12345", icsa.Aliases);
Assert.Contains(icsa.References, reference => reference.Url == "https://example.com/security/icsa-25-123-01");
Assert.Contains(icsa.References, reference => reference.Url == "https://files.cisa.gov/docs/icsa-25-123-01.pdf" && reference.Kind == "attachment");
@@ -88,7 +87,7 @@ public sealed class IcsCisaConnectorTests : IAsyncLifetime
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton(_handler);
services.AddMongoStorage(options =>

View File

@@ -0,0 +1,76 @@
<!DOCTYPE html>
<html lang="ko">
<head>
<meta charset="utf-8" />
<title>국내 취약점 정보</title>
</head>
<body>
<div class="domestic_contents">
<table class="basicView">
<tbody>
<tr>
<td class="bg_tht" colspan="2">
CVE-2025-29866 | 태그프리 제품 부적절한 권한 검증 취약점
<span class="date">2025.07.31</span>
</td>
</tr>
<tr>
<td class="cont" colspan="2">
<p>
<span>□ 개요</span><br />
<span> o 태그프리社의 X-Free Uploader에서 발생하는 부적절한 권한 검증 취약점</span>
</p>
<table class="severity">
<tbody>
<tr>
<td>취약점 종류</td>
<td>영향</td>
<td>심각도</td>
<td>CVSS</td>
<td>CVE ID</td>
</tr>
<tr>
<td>부적절한 권한 검증</td>
<td>데이터 변조</td>
<td>High</td>
<td>8.8</td>
<td>CVE-2025-29866</td>
</tr>
</tbody>
</table>
<p>
<span>□ 영향받는 제품 및 해결 방안</span>
</p>
<table class="product">
<tbody>
<tr>
<td>제품</td>
<td>영향받는 버전</td>
<td>해결 버전</td>
</tr>
<tr>
<td rowspan="2">TAGFREE X-Free Uploader</td>
<td>{{PRIMARY_VERSION}}</td>
<td>XFU 1.0.1.0085</td>
</tr>
<tr>
<td>{{SECONDARY_VERSION}}</td>
<td>XFU 2.0.1.0035</td>
</tr>
</tbody>
</table>
<p>
<span>□ 참고사이트</span>
</p>
<p>
<a href="https://www.tagfree.com/bbs/board.php?bo_table=wb_xfu_update">
https://www.tagfree.com/bbs/board.php?bo_table=wb_xfu_update
</a>
</p>
</td>
</tr>
</tbody>
</table>
</div>
</body>
</html>

View File

@@ -1,213 +1,497 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Connector.Kisa.Configuration;
using StellaOps.Concelier.Connector.Kisa.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Testing;
using Xunit;
using System.Linq;
namespace StellaOps.Concelier.Connector.Kisa.Tests;
[Collection("mongo-fixture")]
public sealed class KisaConnectorTests : IAsyncLifetime
{
private static readonly Uri FeedUri = new("https://test.local/rss/securityInfo.do");
private static readonly Uri DetailApiUri = new("https://test.local/rssDetailData.do?IDX=5868");
private static readonly Uri DetailPageUri = new("https://test.local/detailDos.do?IDX=5868");
private readonly MongoIntegrationFixture _fixture;
private readonly CannedHttpMessageHandler _handler;
public KisaConnectorTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
_handler = new CannedHttpMessageHandler();
}
[Fact]
public async Task FetchParseMap_ProducesCanonicalAdvisory()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses();
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
advisories.Should().HaveCount(1);
var advisory = advisories[0];
advisory.AdvisoryKey.Should().Be("5868");
advisory.Language.Should().Be("ko");
advisory.Aliases.Should().Contain("CVE-2025-29866");
advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("태그프리"));
advisory.References.Should().Contain(reference => reference.Url == DetailPageUri.ToString());
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(KisaConnectorPlugin.SourceName, CancellationToken.None);
state.Should().NotBeNull();
state!.Cursor.Should().NotBeNull();
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
pendingDocs!.AsBsonArray.Should().BeEmpty();
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
pendingMappings!.AsBsonArray.Should().BeEmpty();
}
[Fact]
public async Task Telemetry_RecordsMetrics()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses();
using var metrics = new KisaMetricCollector();
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
Sum(metrics.Measurements, "kisa.feed.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.feed.items").Should().BeGreaterThan(0);
Sum(metrics.Measurements, "kisa.detail.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.detail.failures").Should().Be(0);
Sum(metrics.Measurements, "kisa.parse.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.parse.failures").Should().Be(0);
Sum(metrics.Measurements, "kisa.map.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.map.failures").Should().Be(0);
}
private async Task<ServiceProvider> BuildServiceProviderAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddKisaConnector(options =>
{
options.FeedUri = FeedUri;
options.DetailApiUri = new Uri("https://test.local/rssDetailData.do");
options.DetailPageUri = new Uri("https://test.local/detailDos.do");
options.RequestDelay = TimeSpan.Zero;
options.MaxAdvisoriesPerFetch = 10;
options.MaxKnownAdvisories = 32;
});
services.Configure<HttpClientFactoryOptions>(KisaOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
{
builder.PrimaryHandler = _handler;
});
});
var provider = services.BuildServiceProvider();
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
return provider;
}
private void SeedResponses()
{
AddXmlResponse(FeedUri, ReadFixture("kisa-feed.xml"));
AddJsonResponse(DetailApiUri, ReadFixture("kisa-detail.json"));
}
private void AddXmlResponse(Uri uri, string xml)
{
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(xml, Encoding.UTF8, "application/rss+xml"),
});
}
private void AddJsonResponse(Uri uri, string json)
{
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(json, Encoding.UTF8, "application/json"),
});
}
private static string ReadFixture(string fileName)
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
private static long Sum(IEnumerable<KisaMetricCollector.MetricMeasurement> measurements, string name)
=> measurements.Where(m => m.Name == name).Sum(m => m.Value);
private sealed class KisaMetricCollector : IDisposable
{
private readonly MeterListener _listener;
private readonly ConcurrentBag<MetricMeasurement> _measurements = new();
public KisaMetricCollector()
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == KisaDiagnostics.MeterName)
{
listener.EnableMeasurementEvents(instrument);
}
},
};
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
var tagList = new List<KeyValuePair<string, object?>>(tags.Length);
foreach (var tag in tags)
{
tagList.Add(tag);
}
_measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList));
});
_listener.Start();
}
public IReadOnlyCollection<MetricMeasurement> Measurements => _measurements;
public void Dispose() => _listener.Dispose();
internal sealed record MetricMeasurement(string Name, long Value, IReadOnlyList<KeyValuePair<string, object?>> Tags);
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => Task.CompletedTask;
}
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Connector.Kisa.Configuration;
using StellaOps.Concelier.Connector.Kisa.Internal;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Testing;
using Xunit;
using Xunit.Abstractions;
namespace StellaOps.Concelier.Connector.Kisa.Tests;
[Collection("mongo-fixture")]
public sealed class KisaConnectorTests : IAsyncLifetime
{
private static readonly Uri FeedUri = new("https://test.local/rss/securityInfo.do");
private static readonly Uri DetailPageUri = new("https://test.local/detailDos.do?IDX=5868");
private readonly MongoIntegrationFixture _fixture;
private readonly CannedHttpMessageHandler _handler;
private readonly ITestOutputHelper _output;
public KisaConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output)
{
_fixture = fixture;
_handler = new CannedHttpMessageHandler();
_output = output;
}
[Fact]
public async Task FetchParseMap_ProducesCanonicalAdvisory()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses();
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
advisories.Should().HaveCount(1);
var advisory = advisories[0];
advisory.AdvisoryKey.Should().Be("5868");
advisory.Language.Should().Be("ko");
advisory.Aliases.Should().Contain("CVE-2025-29866");
advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("태그프리"));
advisory.References.Should().Contain(reference => reference.Url == DetailPageUri.ToString());
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Scheme.Should().Be(NormalizedVersionSchemes.SemVer);
normalized.Type.Should().Be(NormalizedVersionRuleTypes.Range);
normalized.Min.Should().Be("1.0.1-fw.84");
normalized.MinInclusive.Should().BeTrue();
normalized.Max.Should().Be("2.0.1-fw.34");
normalized.MaxInclusive.Should().BeTrue();
package.VersionRanges.Should().ContainSingle();
var range = package.VersionRanges.Single();
range.RangeKind.Should().Be("product");
range.RangeExpression.Should().Be("XFU 1.0.1.0084 ~ 2.0.1.0034");
var semVer = GetSemVer(range.Primitives);
semVer.Introduced.Should().Be("1.0.1-fw.84");
semVer.IntroducedInclusive.Should().BeTrue();
semVer.Fixed.Should().Be("2.0.1-fw.34");
semVer.FixedInclusive.Should().BeTrue();
semVer.ConstraintExpression.Should().Be(">= 1.0.1-fw.84 <= 2.0.1-fw.34");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions.Should().ContainKey("kisa.range.raw").WhoseValue.Should().Be("XFU 1.0.1.0084 ~ 2.0.1.0034");
vendorExtensions.Should().ContainKey("kisa.range.prefix").WhoseValue.Should().Be("XFU");
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(KisaConnectorPlugin.SourceName, CancellationToken.None);
state.Should().NotBeNull();
state!.Cursor.Should().NotBeNull();
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
pendingDocs!.AsBsonArray.Should().BeEmpty();
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
pendingMappings!.AsBsonArray.Should().BeEmpty();
}
[Fact]
public async Task FetchParseMap_ExclusiveUpperBound_ProducesExclusiveNormalizedRule()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("XFU 3.2 이상 4.0 미만");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Min.Should().Be("3.2.0");
normalized.MinInclusive.Should().BeTrue();
normalized.Max.Should().Be("4.0.0");
normalized.MaxInclusive.Should().BeFalse();
var range = package.VersionRanges.Single();
var semVer = GetSemVer(range.Primitives);
semVer.FixedInclusive.Should().BeFalse();
semVer.ConstraintExpression.Should().Be(">= 3.2.0 < 4.0.0");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.normalized")
.WhoseValue.Should().Be(">= 3.2.0 < 4.0.0");
}
[Fact]
public async Task FetchParseMap_ExclusiveLowerBound_ProducesExclusiveNormalizedRule()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("XFU 1.2.0 초과 2.4.0 이하");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Min.Should().Be("1.2.0");
normalized.MinInclusive.Should().BeFalse();
normalized.Max.Should().Be("2.4.0");
normalized.MaxInclusive.Should().BeTrue();
var range = package.VersionRanges.Single();
var semVer = GetSemVer(range.Primitives);
semVer.IntroducedInclusive.Should().BeFalse();
semVer.FixedInclusive.Should().BeTrue();
semVer.ConstraintExpression.Should().Be("> 1.2.0 <= 2.4.0");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.normalized")
.WhoseValue.Should().Be("> 1.2.0 <= 2.4.0");
}
[Fact]
public async Task FetchParseMap_SingleBound_ProducesMinimumOnlyConstraint()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("XFU 5.0 이상");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Min.Should().Be("5.0.0");
normalized.MinInclusive.Should().BeTrue();
normalized.Type.Should().Be(NormalizedVersionRuleTypes.GreaterThanOrEqual);
normalized.Max.Should().BeNull();
normalized.MaxInclusive.Should().BeNull();
_output.WriteLine($"normalized: scheme={normalized.Scheme}, type={normalized.Type}, min={normalized.Min}, minInclusive={normalized.MinInclusive}, max={normalized.Max}, maxInclusive={normalized.MaxInclusive}, notes={normalized.Notes}");
var range = package.VersionRanges.Single();
var semVer = GetSemVer(range.Primitives);
semVer.Introduced.Should().Be("5.0.0");
semVer.Fixed.Should().BeNull();
semVer.LastAffected.Should().BeNull();
semVer.ConstraintExpression.Should().Be(">= 5.0.0");
_output.WriteLine($"semver: introduced={semVer.Introduced}, introducedInclusive={semVer.IntroducedInclusive}, fixed={semVer.Fixed}, fixedInclusive={semVer.FixedInclusive}, lastAffected={semVer.LastAffected}, lastAffectedInclusive={semVer.LastAffectedInclusive}, constraint={semVer.ConstraintExpression}");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.normalized")
.WhoseValue.Should().Be(">= 5.0.0");
}
[Fact]
public async Task FetchParseMap_UpperBoundOnlyExclusive_ProducesLessThanRule()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("XFU 3.5 미만");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Type.Should().Be(NormalizedVersionRuleTypes.LessThan);
normalized.Min.Should().BeNull();
normalized.Max.Should().Be("3.5.0");
normalized.MaxInclusive.Should().BeFalse();
var range = package.VersionRanges.Single();
var semVer = GetSemVer(range.Primitives);
semVer.Fixed.Should().Be("3.5.0");
semVer.FixedInclusive.Should().BeFalse();
semVer.ConstraintExpression.Should().Be("< 3.5.0");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.normalized")
.WhoseValue.Should().Be("< 3.5.0");
}
[Fact]
public async Task FetchParseMap_UpperBoundOnlyInclusive_ProducesLessThanOrEqualRule()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("XFU 4.2 이하");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Type.Should().Be(NormalizedVersionRuleTypes.LessThanOrEqual);
normalized.Max.Should().Be("4.2.0");
normalized.MaxInclusive.Should().BeTrue();
var range = package.VersionRanges.Single();
var semVer = GetSemVer(range.Primitives);
semVer.Fixed.Should().Be("4.2.0");
semVer.FixedInclusive.Should().BeTrue();
semVer.ConstraintExpression.Should().Be("<= 4.2.0");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.normalized")
.WhoseValue.Should().Be("<= 4.2.0");
}
[Fact]
public async Task FetchParseMap_LowerBoundOnlyExclusive_ProducesGreaterThanRule()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("XFU 1.9 초과");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
var normalized = GetSingleNormalizedVersion(package);
normalized.Type.Should().Be(NormalizedVersionRuleTypes.GreaterThan);
normalized.Min.Should().Be("1.9.0");
normalized.MinInclusive.Should().BeFalse();
normalized.Max.Should().BeNull();
var range = package.VersionRanges.Single();
var semVer = GetSemVer(range.Primitives);
semVer.Introduced.Should().Be("1.9.0");
semVer.IntroducedInclusive.Should().BeFalse();
semVer.ConstraintExpression.Should().Be("> 1.9.0");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.normalized")
.WhoseValue.Should().Be("> 1.9.0");
}
[Fact]
public async Task FetchParseMap_InvalidSegment_ProducesFallbackRange()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses("지원 버전: 최신 업데이트 적용");
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisory = (await advisoryStore.GetRecentAsync(1, CancellationToken.None)).Single();
var package = advisory.AffectedPackages.Single();
package.NormalizedVersions.Should().BeEmpty();
var range = package.VersionRanges.Single();
range.RangeKind.Should().Be("string");
range.RangeExpression.Should().Be("지원 버전: 최신 업데이트 적용");
var vendorExtensions = GetVendorExtensions(range.Primitives);
vendorExtensions
.Should().ContainKey("kisa.range.raw")
.WhoseValue.Should().Be("지원 버전: 최신 업데이트 적용");
}
[Fact]
public async Task Telemetry_RecordsMetrics()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses();
using var metrics = new KisaMetricCollector();
var connector = provider.GetRequiredService<KisaConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
Sum(metrics.Measurements, "kisa.feed.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.feed.items").Should().BeGreaterThan(0);
Sum(metrics.Measurements, "kisa.detail.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.detail.failures").Should().Be(0);
Sum(metrics.Measurements, "kisa.parse.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.parse.failures").Should().Be(0);
Sum(metrics.Measurements, "kisa.map.success").Should().Be(1);
Sum(metrics.Measurements, "kisa.map.failures").Should().Be(0);
}
private static NormalizedVersionRule GetSingleNormalizedVersion(AffectedPackage package)
{
var normalizedVersions = package.NormalizedVersions;
if (normalizedVersions.IsDefaultOrEmpty)
{
throw new InvalidOperationException("Expected normalized version rule.");
}
return normalizedVersions.Single();
}
private static SemVerPrimitive GetSemVer(RangePrimitives? primitives)
=> primitives?.SemVer ?? throw new InvalidOperationException("Expected semver primitive.");
private static IReadOnlyDictionary<string, string> GetVendorExtensions(RangePrimitives? primitives)
=> primitives?.VendorExtensions ?? throw new InvalidOperationException("Expected vendor extensions.");
private async Task<ServiceProvider> BuildServiceProviderAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddKisaConnector(options =>
{
options.FeedUri = FeedUri;
options.DetailApiUri = new Uri("https://test.local/rssDetailData.do");
options.DetailPageUri = new Uri("https://test.local/detailDos.do");
options.RequestDelay = TimeSpan.Zero;
options.MaxAdvisoriesPerFetch = 10;
options.MaxKnownAdvisories = 32;
});
services.Configure<HttpClientFactoryOptions>(KisaOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
{
builder.PrimaryHandler = _handler;
});
});
var provider = services.BuildServiceProvider();
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
return provider;
}
private void SeedResponses(string? versionOverride = null)
{
AddXmlResponse(FeedUri, ReadFixture("kisa-feed.xml"));
var detailPayload = BuildDetailHtml(versionOverride);
AddHtmlResponse(DetailPageUri, detailPayload);
}
private void AddXmlResponse(Uri uri, string xml)
{
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(xml, Encoding.UTF8, "application/rss+xml"),
});
}
private void AddHtmlResponse(Uri uri, string html)
{
_handler.AddResponse(uri, () => new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(html, Encoding.UTF8, "text/html"),
});
}
private static string ReadFixture(string fileName)
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
private static string BuildDetailHtml(string? versions)
{
var template = ReadFixture("kisa-detail.html");
var primary = versions ?? "XFU 1.0.1.0084";
var secondary = versions is null ? "XFU 2.0.1.0034" : string.Empty;
return template
.Replace("{{PRIMARY_VERSION}}", primary, StringComparison.Ordinal)
.Replace("{{SECONDARY_VERSION}}", secondary, StringComparison.Ordinal);
}
private static long Sum(IEnumerable<KisaMetricCollector.MetricMeasurement> measurements, string name)
=> measurements.Where(m => m.Name == name).Sum(m => m.Value);
private sealed class KisaMetricCollector : IDisposable
{
private readonly MeterListener _listener;
private readonly ConcurrentBag<MetricMeasurement> _measurements = new();
public KisaMetricCollector()
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == KisaDiagnostics.MeterName)
{
listener.EnableMeasurementEvents(instrument);
}
},
};
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
{
var tagList = new List<KeyValuePair<string, object?>>(tags.Length);
foreach (var tag in tags)
{
tagList.Add(tag);
}
_measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList));
});
_listener.Start();
}
public IReadOnlyCollection<MetricMeasurement> Measurements => _measurements;
public void Dispose() => _listener.Dispose();
internal sealed record MetricMeasurement(string Name, long Value, IReadOnlyList<KeyValuePair<string, object?>> Tags);
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => Task.CompletedTask;
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using FluentAssertions;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Connector.Kisa.Internal;
using Xunit;
namespace StellaOps.Concelier.Connector.Kisa.Tests;
public sealed class KisaDetailParserTests
{
private static readonly Uri DetailApiUri = new("https://test.local/rssDetailData.do?IDX=5868");
private static readonly Uri DetailPageUri = new("https://test.local/detailDos.do?IDX=5868");
[Fact]
public void ParseHtmlPayload_ProducesExpectedModels()
{
var parser = new KisaDetailParser(new HtmlContentSanitizer());
var payload = ReadFixtureBytes("kisa-detail.html", "XFU 1.0.1.0084", "XFU 2.0.1.0034");
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["kisa.idx"] = "5868",
["kisa.title"] = "태그프리 제품 부적절한 권한 검증 취약점",
["kisa.published"] = "2025-07-31T06:30:23Z",
};
var parsed = parser.Parse(DetailApiUri, DetailPageUri, payload, metadata);
parsed.AdvisoryId.Should().Be("5868");
parsed.Title.Should().Contain("태그프리");
parsed.Summary.Should().NotBeNullOrWhiteSpace();
parsed.ContentHtml.Should().Contain("TAGFREE");
parsed.Severity.Should().Be("High");
parsed.CveIds.Should().Contain("CVE-2025-29866");
parsed.Products.Should().ContainSingle();
var product = parsed.Products.Single();
product.Vendor.Should().Be("태그프리");
product.Name.Should().Be("X-Free Uploader");
product.Versions.Should().Be("XFU 1.0.1.0084 ~ 2.0.1.0034");
}
private static byte[] ReadFixtureBytes(string fileName, string primaryVersion, string secondaryVersion)
{
var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName);
var template = File.ReadAllText(path);
var html = template
.Replace("{{PRIMARY_VERSION}}", primaryVersion, StringComparison.Ordinal)
.Replace("{{SECONDARY_VERSION}}", secondaryVersion, StringComparison.Ordinal);
return Encoding.UTF8.GetBytes(html);
}
}

View File

@@ -13,6 +13,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.14.1" />
</ItemGroup>
<ItemGroup>
<None Update="Fixtures/*.json">
@@ -21,5 +22,8 @@
<None Update="Fixtures/*.xml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Fixtures/*.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
</Project>