Rename Concelier Source modules to Connector
This commit is contained in:
@@ -0,0 +1,418 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Normalization.Cvss;
|
||||
using StellaOps.Concelier.Normalization.Identifiers;
|
||||
using StellaOps.Concelier.Normalization.Text;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.JpFlags;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal static class JvnAdvisoryMapper
|
||||
{
|
||||
private static readonly string[] SeverityOrder = { "none", "low", "medium", "high", "critical" };
|
||||
|
||||
public static (Advisory Advisory, JpFlagRecord Flag) Map(
|
||||
JvnDetailDto detail,
|
||||
DocumentRecord document,
|
||||
DtoRecord dtoRecord,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(detail);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
var recordedAt = dtoRecord.ValidatedAt;
|
||||
var fetchProvenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt);
|
||||
var mappingProvenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "mapping", detail.VulnerabilityId, recordedAt);
|
||||
|
||||
var aliases = BuildAliases(detail);
|
||||
var references = BuildReferences(detail, recordedAt);
|
||||
var affectedPackages = BuildAffected(detail, recordedAt);
|
||||
var cvssMetrics = BuildCvss(detail, recordedAt, out var severity);
|
||||
|
||||
var description = DescriptionNormalizer.Normalize(new[]
|
||||
{
|
||||
new LocalizedText(detail.Overview, detail.Language),
|
||||
});
|
||||
|
||||
var language = description.Language;
|
||||
var summary = string.IsNullOrEmpty(description.Text) ? null : description.Text;
|
||||
|
||||
var provenance = new[] { fetchProvenance, mappingProvenance };
|
||||
|
||||
var advisory = new Advisory(
|
||||
detail.VulnerabilityId,
|
||||
detail.Title,
|
||||
summary,
|
||||
language,
|
||||
detail.DateFirstPublished,
|
||||
detail.DateLastUpdated,
|
||||
severity,
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
references,
|
||||
affectedPackages,
|
||||
cvssMetrics,
|
||||
provenance);
|
||||
|
||||
var vendorStatus = detail.VendorStatuses.Length == 0
|
||||
? null
|
||||
: string.Join(",", detail.VendorStatuses.OrderBy(static status => status, StringComparer.Ordinal));
|
||||
|
||||
var flag = new JpFlagRecord(
|
||||
detail.VulnerabilityId,
|
||||
JvnConnectorPlugin.SourceName,
|
||||
detail.JvnCategory,
|
||||
vendorStatus,
|
||||
timeProvider.GetUtcNow());
|
||||
|
||||
return (advisory, flag);
|
||||
}
|
||||
|
||||
private static IEnumerable<string> BuildAliases(JvnDetailDto detail)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
detail.VulnerabilityId,
|
||||
};
|
||||
|
||||
foreach (var cve in detail.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
return aliases;
|
||||
}
|
||||
|
||||
private static IEnumerable<AdvisoryReference> BuildReferences(JvnDetailDto detail, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
|
||||
foreach (var reference in detail.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string? kind = reference.Type?.ToLowerInvariant() switch
|
||||
{
|
||||
"vendor" => "vendor",
|
||||
"advisory" => "advisory",
|
||||
"cwe" => "weakness",
|
||||
_ => null,
|
||||
};
|
||||
|
||||
string? sourceTag = !string.IsNullOrWhiteSpace(reference.Id) ? reference.Id : reference.Type;
|
||||
string? summary = reference.Name;
|
||||
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
kind,
|
||||
sourceTag,
|
||||
summary,
|
||||
new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "reference", reference.Url, recordedAt)));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// Ignore malformed URLs that slipped through validation.
|
||||
}
|
||||
}
|
||||
|
||||
if (references.Count == 0)
|
||||
{
|
||||
return references;
|
||||
}
|
||||
|
||||
var map = new Dictionary<string, AdvisoryReference>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var reference in references)
|
||||
{
|
||||
if (!map.TryGetValue(reference.Url, out var existing))
|
||||
{
|
||||
map[reference.Url] = reference;
|
||||
continue;
|
||||
}
|
||||
|
||||
map[reference.Url] = MergeReferences(existing, reference);
|
||||
}
|
||||
|
||||
var deduped = map.Values.ToList();
|
||||
deduped.Sort(CompareReferences);
|
||||
return deduped;
|
||||
}
|
||||
|
||||
private static IEnumerable<AffectedPackage> BuildAffected(JvnDetailDto detail, DateTimeOffset recordedAt)
|
||||
{
|
||||
var packages = new List<AffectedPackage>();
|
||||
|
||||
foreach (var product in detail.Affected)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.Cpe))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Status) && !product.Status.StartsWith("vulnerable", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!IdentifierNormalizer.TryNormalizeCpe(product.Cpe, out var cpe))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new List<AdvisoryProvenance>
|
||||
{
|
||||
new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "affected", cpe!, recordedAt),
|
||||
};
|
||||
|
||||
var attributeParts = new List<string>(capacity: 2);
|
||||
if (!string.IsNullOrWhiteSpace(product.CpeVendor))
|
||||
{
|
||||
attributeParts.Add($"vendor={product.CpeVendor}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.CpeProduct))
|
||||
{
|
||||
attributeParts.Add($"product={product.CpeProduct}");
|
||||
}
|
||||
|
||||
if (attributeParts.Count > 0)
|
||||
{
|
||||
provenance.Add(new AdvisoryProvenance(
|
||||
JvnConnectorPlugin.SourceName,
|
||||
"cpe-attributes",
|
||||
string.Join(";", attributeParts),
|
||||
recordedAt));
|
||||
}
|
||||
|
||||
var platform = product.Vendor ?? product.CpeVendor;
|
||||
|
||||
var versionRanges = BuildVersionRanges(product, recordedAt, provenance[0]);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Cpe,
|
||||
cpe!,
|
||||
platform: platform,
|
||||
versionRanges: versionRanges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance.ToArray()));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(JvnAffectedProductDto product, DateTimeOffset recordedAt, AdvisoryProvenance provenance)
|
||||
{
|
||||
var extensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (!string.IsNullOrWhiteSpace(product.Version))
|
||||
{
|
||||
extensions["jvn.version"] = product.Version!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Build))
|
||||
{
|
||||
extensions["jvn.build"] = product.Build!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Description))
|
||||
{
|
||||
extensions["jvn.description"] = product.Description!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Status))
|
||||
{
|
||||
extensions["jvn.status"] = product.Status!;
|
||||
}
|
||||
|
||||
if (extensions.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedVersionRange>();
|
||||
}
|
||||
|
||||
var primitives = new RangePrimitives(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
extensions);
|
||||
|
||||
var expression = product.Version;
|
||||
var range = new AffectedVersionRange(
|
||||
rangeKind: "cpe",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: string.IsNullOrWhiteSpace(expression) ? null : expression,
|
||||
provenance: provenance,
|
||||
primitives: primitives);
|
||||
|
||||
return new[] { range };
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CvssMetric> BuildCvss(JvnDetailDto detail, DateTimeOffset recordedAt, out string? severity)
|
||||
{
|
||||
var metrics = new List<CvssMetric>();
|
||||
severity = null;
|
||||
var bestRank = -1;
|
||||
|
||||
foreach (var cvss in detail.Cvss)
|
||||
{
|
||||
if (!CvssMetricNormalizer.TryNormalize(cvss.Version, cvss.Vector, cvss.Score, cvss.Severity, out var normalized))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "cvss", cvss.Type, recordedAt);
|
||||
metrics.Add(normalized.ToModel(provenance));
|
||||
|
||||
var rank = Array.IndexOf(SeverityOrder, normalized.BaseSeverity);
|
||||
if (rank > bestRank)
|
||||
{
|
||||
bestRank = rank;
|
||||
severity = normalized.BaseSeverity;
|
||||
}
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static int CompareReferences(AdvisoryReference? left, AdvisoryReference? right)
|
||||
{
|
||||
if (ReferenceEquals(left, right))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (left is null)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (right is null)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
var compare = StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
compare = CompareNullable(left.Kind, right.Kind);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
compare = CompareNullable(left.SourceTag, right.SourceTag);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
compare = CompareNullable(left.Summary, right.Summary);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
compare = StringComparer.Ordinal.Compare(left.Provenance.Source, right.Provenance.Source);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
compare = StringComparer.Ordinal.Compare(left.Provenance.Kind, right.Provenance.Kind);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
compare = CompareNullable(left.Provenance.Value, right.Provenance.Value);
|
||||
if (compare != 0)
|
||||
{
|
||||
return compare;
|
||||
}
|
||||
|
||||
return left.Provenance.RecordedAt.CompareTo(right.Provenance.RecordedAt);
|
||||
}
|
||||
|
||||
private static int CompareNullable(string? left, string? right)
|
||||
{
|
||||
if (left is null && right is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (left is null)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (right is null)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
return StringComparer.Ordinal.Compare(left, right);
|
||||
}
|
||||
|
||||
private static AdvisoryReference MergeReferences(AdvisoryReference existing, AdvisoryReference candidate)
|
||||
{
|
||||
var kind = existing.Kind ?? candidate.Kind;
|
||||
var sourceTag = existing.SourceTag ?? candidate.SourceTag;
|
||||
var summary = ChoosePreferredSummary(existing.Summary, candidate.Summary);
|
||||
var provenance = existing.Provenance.RecordedAt <= candidate.Provenance.RecordedAt
|
||||
? existing.Provenance
|
||||
: candidate.Provenance;
|
||||
|
||||
if (kind == existing.Kind
|
||||
&& sourceTag == existing.SourceTag
|
||||
&& summary == existing.Summary
|
||||
&& provenance == existing.Provenance)
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (kind == candidate.Kind
|
||||
&& sourceTag == candidate.SourceTag
|
||||
&& summary == candidate.Summary
|
||||
&& provenance == candidate.Provenance)
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
|
||||
return new AdvisoryReference(existing.Url, kind, sourceTag, summary, provenance);
|
||||
}
|
||||
|
||||
private static string? ChoosePreferredSummary(string? left, string? right)
|
||||
{
|
||||
var leftValue = string.IsNullOrWhiteSpace(left) ? null : left;
|
||||
var rightValue = string.IsNullOrWhiteSpace(right) ? null : right;
|
||||
|
||||
if (leftValue is null)
|
||||
{
|
||||
return rightValue;
|
||||
}
|
||||
|
||||
if (rightValue is null)
|
||||
{
|
||||
return leftValue;
|
||||
}
|
||||
|
||||
return leftValue.Length >= rightValue.Length ? leftValue : rightValue;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal static class JvnConstants
|
||||
{
|
||||
public const string DtoSchemaVersion = "jvn.vuldef.3.2";
|
||||
|
||||
public const string VuldefNamespace = "http://jvn.jp/vuldef/";
|
||||
public const string StatusNamespace = "http://jvndb.jvn.jp/myjvn/Status";
|
||||
public const string ModSecNamespace = "http://jvn.jp/rss/mod_sec/3.0/";
|
||||
}
|
||||
106
src/StellaOps.Concelier.Connector.Jvn/Internal/JvnCursor.cs
Normal file
106
src/StellaOps.Concelier.Connector.Jvn/Internal/JvnCursor.cs
Normal file
@@ -0,0 +1,106 @@
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal sealed record JvnCursor(
|
||||
DateTimeOffset? WindowStart,
|
||||
DateTimeOffset? WindowEnd,
|
||||
DateTimeOffset? LastCompletedWindowEnd,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
public static JvnCursor Empty { get; } = new(null, null, null, Array.Empty<Guid>(), Array.Empty<Guid>());
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument();
|
||||
|
||||
if (WindowStart.HasValue)
|
||||
{
|
||||
document["windowStart"] = WindowStart.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (WindowEnd.HasValue)
|
||||
{
|
||||
document["windowEnd"] = WindowEnd.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (LastCompletedWindowEnd.HasValue)
|
||||
{
|
||||
document["lastCompletedWindowEnd"] = LastCompletedWindowEnd.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
|
||||
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
|
||||
return document;
|
||||
}
|
||||
|
||||
public static JvnCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
DateTimeOffset? windowStart = TryGetDateTime(document, "windowStart");
|
||||
DateTimeOffset? windowEnd = TryGetDateTime(document, "windowEnd");
|
||||
DateTimeOffset? lastCompleted = TryGetDateTime(document, "lastCompletedWindowEnd");
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
return new JvnCursor(windowStart, windowEnd, lastCompleted, pendingDocuments, pendingMappings);
|
||||
}
|
||||
|
||||
public JvnCursor WithWindow(DateTimeOffset start, DateTimeOffset end)
|
||||
=> this with { WindowStart = start, WindowEnd = end };
|
||||
|
||||
public JvnCursor WithCompletedWindow(DateTimeOffset end)
|
||||
=> this with { LastCompletedWindowEnd = end };
|
||||
|
||||
public JvnCursor WithPendingDocuments(IEnumerable<Guid> pending)
|
||||
=> this with { PendingDocuments = pending?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public JvnCursor WithPendingMappings(IEnumerable<Guid> pending)
|
||||
=> this with { PendingMappings = pending?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
private static DateTimeOffset? TryGetDateTime(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (element.BsonType == BsonType.String && Guid.TryParse(element.AsString, out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal sealed record JvnDetailDto(
|
||||
string VulnerabilityId,
|
||||
string Title,
|
||||
string? Overview,
|
||||
string? Language,
|
||||
DateTimeOffset? DateFirstPublished,
|
||||
DateTimeOffset? DateLastUpdated,
|
||||
DateTimeOffset? DatePublic,
|
||||
ImmutableArray<JvnCvssDto> Cvss,
|
||||
ImmutableArray<JvnAffectedProductDto> Affected,
|
||||
ImmutableArray<JvnReferenceDto> References,
|
||||
ImmutableArray<JvnHistoryEntryDto> History,
|
||||
ImmutableArray<string> CweIds,
|
||||
ImmutableArray<string> CveIds,
|
||||
string? AdvisoryUrl,
|
||||
string? JvnCategory,
|
||||
ImmutableArray<string> VendorStatuses)
|
||||
{
|
||||
public static JvnDetailDto Empty { get; } = new(
|
||||
"unknown",
|
||||
"unknown",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ImmutableArray<JvnCvssDto>.Empty,
|
||||
ImmutableArray<JvnAffectedProductDto>.Empty,
|
||||
ImmutableArray<JvnReferenceDto>.Empty,
|
||||
ImmutableArray<JvnHistoryEntryDto>.Empty,
|
||||
ImmutableArray<string>.Empty,
|
||||
ImmutableArray<string>.Empty,
|
||||
null,
|
||||
null,
|
||||
ImmutableArray<string>.Empty);
|
||||
}
|
||||
|
||||
internal sealed record JvnCvssDto(
|
||||
string Version,
|
||||
string Type,
|
||||
string Severity,
|
||||
double Score,
|
||||
string? Vector);
|
||||
|
||||
internal sealed record JvnAffectedProductDto(
|
||||
string? Vendor,
|
||||
string? Product,
|
||||
string? Cpe,
|
||||
string? CpeVendor,
|
||||
string? CpeProduct,
|
||||
string? Version,
|
||||
string? Build,
|
||||
string? Description,
|
||||
string? Status);
|
||||
|
||||
internal sealed record JvnReferenceDto(
|
||||
string Type,
|
||||
string Id,
|
||||
string? Name,
|
||||
string Url);
|
||||
|
||||
internal sealed record JvnHistoryEntryDto(
|
||||
string? Number,
|
||||
DateTimeOffset? Timestamp,
|
||||
string? Description);
|
||||
@@ -0,0 +1,268 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Xml;
|
||||
using System.Xml.Linq;
|
||||
using System.Xml.Schema;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal static class JvnDetailParser
|
||||
{
|
||||
private static readonly XNamespace Vuldef = JvnConstants.VuldefNamespace;
|
||||
private static readonly XNamespace Status = JvnConstants.StatusNamespace;
|
||||
|
||||
public static JvnDetailDto Parse(byte[] payload, string? documentUri)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(payload);
|
||||
|
||||
using var stream = new MemoryStream(payload, writable: false);
|
||||
var settings = new XmlReaderSettings
|
||||
{
|
||||
DtdProcessing = DtdProcessing.Prohibit,
|
||||
IgnoreComments = true,
|
||||
IgnoreProcessingInstructions = true,
|
||||
IgnoreWhitespace = true,
|
||||
};
|
||||
|
||||
using var reader = XmlReader.Create(stream, settings);
|
||||
var document = XDocument.Load(reader, LoadOptions.None);
|
||||
Validate(document, documentUri);
|
||||
return Extract(document, documentUri);
|
||||
}
|
||||
|
||||
private static void Validate(XDocument document, string? documentUri)
|
||||
{
|
||||
void Handler(object? sender, ValidationEventArgs args)
|
||||
{
|
||||
throw new JvnSchemaValidationException(
|
||||
$"JVN schema validation failed for {documentUri ?? "<unknown>"}: {args.Message}",
|
||||
args.Exception ?? new XmlSchemaValidationException(args.Message));
|
||||
}
|
||||
|
||||
document.Validate(JvnSchemaProvider.SchemaSet, Handler, addSchemaInfo: true);
|
||||
}
|
||||
|
||||
private static JvnDetailDto Extract(XDocument document, string? documentUri)
|
||||
{
|
||||
var root = document.Root ?? throw new InvalidOperationException("JVN VULDEF document missing root element.");
|
||||
|
||||
var vulinfo = root.Element(Vuldef + "Vulinfo") ?? throw new InvalidOperationException("Vulinfo element missing.");
|
||||
var vulinfoId = Clean(vulinfo.Element(Vuldef + "VulinfoID")?.Value)
|
||||
?? throw new InvalidOperationException("VulinfoID element missing.");
|
||||
|
||||
var data = vulinfo.Element(Vuldef + "VulinfoData") ?? throw new InvalidOperationException("VulinfoData element missing.");
|
||||
var title = Clean(data.Element(Vuldef + "Title")?.Value) ?? vulinfoId;
|
||||
var overview = Clean(data.Element(Vuldef + "VulinfoDescription")?.Element(Vuldef + "Overview")?.Value);
|
||||
|
||||
var dateFirstPublished = ParseDate(data.Element(Vuldef + "DateFirstPublished")?.Value);
|
||||
var dateLastUpdated = ParseDate(data.Element(Vuldef + "DateLastUpdated")?.Value);
|
||||
var datePublic = ParseDate(data.Element(Vuldef + "DatePublic")?.Value);
|
||||
|
||||
var cvssEntries = ParseCvss(data.Element(Vuldef + "Impact"));
|
||||
var affected = ParseAffected(data.Element(Vuldef + "Affected"));
|
||||
var references = ParseReferences(data.Element(Vuldef + "Related"));
|
||||
var history = ParseHistory(data.Element(Vuldef + "History"));
|
||||
|
||||
var cweIds = references.Where(r => string.Equals(r.Type, "cwe", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(r => r.Id)
|
||||
.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(static id => id!)
|
||||
.ToImmutableArray();
|
||||
|
||||
var cveIds = references.Where(r => string.Equals(r.Type, "advisory", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.IsNullOrWhiteSpace(r.Id)
|
||||
&& r.Id.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(r => r.Id)
|
||||
.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(static id => id!)
|
||||
.ToImmutableArray();
|
||||
|
||||
var language = Clean(root.Attribute(XNamespace.Xml + "lang")?.Value);
|
||||
|
||||
var statusElement = root.Element(Status + "Status");
|
||||
var jvnCategory = Clean(statusElement?.Attribute("category")?.Value);
|
||||
|
||||
var vendorStatuses = affected
|
||||
.Select(a => a.Status)
|
||||
.Where(static status => !string.IsNullOrWhiteSpace(status))
|
||||
.Select(static status => status!.ToLowerInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new JvnDetailDto(
|
||||
vulinfoId,
|
||||
title,
|
||||
overview,
|
||||
language,
|
||||
dateFirstPublished,
|
||||
dateLastUpdated,
|
||||
datePublic,
|
||||
cvssEntries,
|
||||
affected,
|
||||
references,
|
||||
history,
|
||||
cweIds,
|
||||
cveIds,
|
||||
Clean(documentUri),
|
||||
jvnCategory,
|
||||
vendorStatuses);
|
||||
}
|
||||
|
||||
private static ImmutableArray<JvnCvssDto> ParseCvss(XElement? impactElement)
|
||||
{
|
||||
if (impactElement is null)
|
||||
{
|
||||
return ImmutableArray<JvnCvssDto>.Empty;
|
||||
}
|
||||
|
||||
var results = new List<JvnCvssDto>();
|
||||
foreach (var cvssElement in impactElement.Elements(Vuldef + "Cvss"))
|
||||
{
|
||||
var version = Clean(cvssElement.Attribute("version")?.Value) ?? "";
|
||||
var severityElement = cvssElement.Element(Vuldef + "Severity");
|
||||
var severity = Clean(severityElement?.Value) ?? Clean(cvssElement.Attribute("severity")?.Value) ?? string.Empty;
|
||||
var type = Clean(severityElement?.Attribute("type")?.Value) ?? Clean(cvssElement.Attribute("type")?.Value) ?? "base";
|
||||
var scoreText = Clean(cvssElement.Element(Vuldef + "Base")?.Value)
|
||||
?? Clean(cvssElement.Attribute("score")?.Value)
|
||||
?? "0";
|
||||
if (!double.TryParse(scoreText, NumberStyles.Float, CultureInfo.InvariantCulture, out var score))
|
||||
{
|
||||
score = 0d;
|
||||
}
|
||||
|
||||
var vector = Clean(cvssElement.Element(Vuldef + "Vector")?.Value)
|
||||
?? Clean(cvssElement.Attribute("vector")?.Value);
|
||||
|
||||
results.Add(new JvnCvssDto(
|
||||
version,
|
||||
type,
|
||||
severity,
|
||||
score,
|
||||
vector));
|
||||
}
|
||||
|
||||
return results.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<JvnAffectedProductDto> ParseAffected(XElement? affectedElement)
|
||||
{
|
||||
if (affectedElement is null)
|
||||
{
|
||||
return ImmutableArray<JvnAffectedProductDto>.Empty;
|
||||
}
|
||||
|
||||
var results = new List<JvnAffectedProductDto>();
|
||||
foreach (var item in affectedElement.Elements(Vuldef + "AffectedItem"))
|
||||
{
|
||||
var vendor = Clean(item.Element(Vuldef + "Name")?.Value);
|
||||
var product = Clean(item.Element(Vuldef + "ProductName")?.Value);
|
||||
var cpeElement = item.Element(Vuldef + "Cpe");
|
||||
var cpe = Clean(cpeElement?.Value);
|
||||
var cpeVendor = Clean(cpeElement?.Attribute("vendor")?.Value);
|
||||
var cpeProduct = Clean(cpeElement?.Attribute("product")?.Value);
|
||||
var version = Clean(ReadConcatenated(item.Elements(Vuldef + "VersionNumber")));
|
||||
var build = Clean(ReadConcatenated(item.Elements(Vuldef + "BuildNumber")));
|
||||
var description = Clean(ReadConcatenated(item.Elements(Vuldef + "Description")));
|
||||
var status = Clean(item.Attribute("affectedstatus")?.Value);
|
||||
|
||||
results.Add(new JvnAffectedProductDto(vendor, product, cpe, cpeVendor, cpeProduct, version, build, description, status));
|
||||
}
|
||||
|
||||
return results.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<JvnReferenceDto> ParseReferences(XElement? relatedElement)
|
||||
{
|
||||
if (relatedElement is null)
|
||||
{
|
||||
return ImmutableArray<JvnReferenceDto>.Empty;
|
||||
}
|
||||
|
||||
var results = new List<JvnReferenceDto>();
|
||||
foreach (var item in relatedElement.Elements(Vuldef + "RelatedItem"))
|
||||
{
|
||||
var type = Clean(item.Attribute("type")?.Value) ?? string.Empty;
|
||||
var id = Clean(item.Element(Vuldef + "VulinfoID")?.Value) ?? string.Empty;
|
||||
var name = Clean(item.Element(Vuldef + "Name")?.Value);
|
||||
var url = Clean(item.Element(Vuldef + "URL")?.Value);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || (uri.Scheme is not "http" and not "https"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
results.Add(new JvnReferenceDto(type, id, name, uri.ToString()));
|
||||
}
|
||||
|
||||
return results.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<JvnHistoryEntryDto> ParseHistory(XElement? historyElement)
|
||||
{
|
||||
if (historyElement is null)
|
||||
{
|
||||
return ImmutableArray<JvnHistoryEntryDto>.Empty;
|
||||
}
|
||||
|
||||
var results = new List<JvnHistoryEntryDto>();
|
||||
foreach (var item in historyElement.Elements(Vuldef + "HistoryItem"))
|
||||
{
|
||||
var number = Clean(item.Element(Vuldef + "HistoryNo")?.Value);
|
||||
var timestamp = ParseDate(item.Element(Vuldef + "DateTime")?.Value);
|
||||
var description = Clean(item.Element(Vuldef + "Description")?.Value);
|
||||
results.Add(new JvnHistoryEntryDto(number, timestamp, description));
|
||||
}
|
||||
|
||||
return results.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)
|
||||
? parsed.ToUniversalTime()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static string? Clean(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.Trim();
|
||||
}
|
||||
|
||||
private static string? ReadConcatenated(IEnumerable<XElement> elements)
|
||||
{
|
||||
var builder = new List<string>();
|
||||
foreach (var element in elements)
|
||||
{
|
||||
var text = element?.Value;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(text.Trim());
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? null : string.Join("; ", builder);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal sealed record JvnOverviewItem(
|
||||
string VulnerabilityId,
|
||||
Uri DetailUri,
|
||||
string Title,
|
||||
DateTimeOffset? DateFirstPublished,
|
||||
DateTimeOffset? DateLastUpdated);
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal sealed record JvnOverviewPage(
|
||||
IReadOnlyList<JvnOverviewItem> Items,
|
||||
int TotalResults,
|
||||
int ReturnedCount,
|
||||
int FirstResultIndex);
|
||||
@@ -0,0 +1,167 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Xml;
|
||||
using System.Xml.Schema;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal static class JvnSchemaProvider
|
||||
{
|
||||
private static readonly Lazy<(XmlSchemaSet SchemaSet, EmbeddedResourceXmlResolver Resolver)> Cached = new(
|
||||
LoadSchemas,
|
||||
LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
public static XmlSchemaSet SchemaSet => Cached.Value.SchemaSet;
|
||||
|
||||
private static (XmlSchemaSet SchemaSet, EmbeddedResourceXmlResolver Resolver) LoadSchemas()
|
||||
{
|
||||
var assembly = typeof(JvnSchemaProvider).GetTypeInfo().Assembly;
|
||||
var resourceMap = CreateResourceMap();
|
||||
var resolver = new EmbeddedResourceXmlResolver(assembly, resourceMap);
|
||||
|
||||
var schemaSet = new XmlSchemaSet
|
||||
{
|
||||
XmlResolver = resolver,
|
||||
};
|
||||
|
||||
AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/vuldef_3.2.xsd");
|
||||
AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/mod_sec_3.0.xsd");
|
||||
AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/status_3.3.xsd");
|
||||
AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/tlp_marking.xsd");
|
||||
AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/data_marking.xsd");
|
||||
|
||||
schemaSet.Compile();
|
||||
return (schemaSet, resolver);
|
||||
}
|
||||
|
||||
private static void AddSchema(XmlSchemaSet set, EmbeddedResourceXmlResolver resolver, string uri)
|
||||
{
|
||||
using var stream = resolver.OpenStream(uri);
|
||||
using var reader = XmlReader.Create(stream, new XmlReaderSettings { XmlResolver = resolver }, uri);
|
||||
set.Add(null, reader);
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> CreateResourceMap()
|
||||
{
|
||||
var baseNamespace = typeof(JvnSchemaProvider).Namespace ?? "StellaOps.Concelier.Connector.Jvn.Internal";
|
||||
var prefix = baseNamespace.Replace(".Internal", string.Empty, StringComparison.Ordinal);
|
||||
|
||||
return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["https://jvndb.jvn.jp/schema/vuldef_3.2.xsd"] = $"{prefix}.Schemas.vuldef_3.2.xsd",
|
||||
["vuldef_3.2.xsd"] = $"{prefix}.Schemas.vuldef_3.2.xsd",
|
||||
["https://jvndb.jvn.jp/schema/mod_sec_3.0.xsd"] = $"{prefix}.Schemas.mod_sec_3.0.xsd",
|
||||
["mod_sec_3.0.xsd"] = $"{prefix}.Schemas.mod_sec_3.0.xsd",
|
||||
["https://jvndb.jvn.jp/schema/status_3.3.xsd"] = $"{prefix}.Schemas.status_3.3.xsd",
|
||||
["status_3.3.xsd"] = $"{prefix}.Schemas.status_3.3.xsd",
|
||||
["https://jvndb.jvn.jp/schema/tlp_marking.xsd"] = $"{prefix}.Schemas.tlp_marking.xsd",
|
||||
["tlp_marking.xsd"] = $"{prefix}.Schemas.tlp_marking.xsd",
|
||||
["https://jvndb.jvn.jp/schema/data_marking.xsd"] = $"{prefix}.Schemas.data_marking.xsd",
|
||||
["data_marking.xsd"] = $"{prefix}.Schemas.data_marking.xsd",
|
||||
["https://www.w3.org/2001/xml.xsd"] = $"{prefix}.Schemas.xml.xsd",
|
||||
["xml.xsd"] = $"{prefix}.Schemas.xml.xsd",
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class EmbeddedResourceXmlResolver : XmlResolver
|
||||
{
|
||||
private readonly Assembly _assembly;
|
||||
private readonly Dictionary<string, string> _resourceMap;
|
||||
|
||||
public EmbeddedResourceXmlResolver(Assembly assembly, Dictionary<string, string> resourceMap)
|
||||
{
|
||||
_assembly = assembly ?? throw new ArgumentNullException(nameof(assembly));
|
||||
_resourceMap = resourceMap ?? throw new ArgumentNullException(nameof(resourceMap));
|
||||
}
|
||||
|
||||
public override ICredentials? Credentials
|
||||
{
|
||||
set { }
|
||||
}
|
||||
|
||||
public Stream OpenStream(string uriOrName)
|
||||
{
|
||||
var resourceName = ResolveResourceName(uriOrName)
|
||||
?? throw new FileNotFoundException($"Schema resource '{uriOrName}' not found in manifest.");
|
||||
|
||||
var stream = _assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream is null)
|
||||
{
|
||||
throw new FileNotFoundException($"Embedded schema '{resourceName}' could not be opened.");
|
||||
}
|
||||
|
||||
return stream;
|
||||
}
|
||||
|
||||
public override object? GetEntity(Uri absoluteUri, string? role, Type? ofObjectToReturn)
|
||||
{
|
||||
if (absoluteUri is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(absoluteUri));
|
||||
}
|
||||
|
||||
var resourceName = ResolveResourceName(absoluteUri.AbsoluteUri)
|
||||
?? ResolveResourceName(absoluteUri.AbsolutePath.TrimStart('/'))
|
||||
?? ResolveResourceName(Path.GetFileName(absoluteUri.AbsolutePath))
|
||||
?? throw new FileNotFoundException($"Schema resource for '{absoluteUri}' not found.");
|
||||
|
||||
var stream = _assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream is null)
|
||||
{
|
||||
throw new FileNotFoundException($"Embedded schema '{resourceName}' could not be opened.");
|
||||
}
|
||||
|
||||
return stream;
|
||||
}
|
||||
|
||||
public override Uri ResolveUri(Uri? baseUri, string? relativeUri)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativeUri))
|
||||
{
|
||||
return base.ResolveUri(baseUri, relativeUri);
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(relativeUri, UriKind.Absolute, out var absolute))
|
||||
{
|
||||
return absolute;
|
||||
}
|
||||
|
||||
if (baseUri is not null && Uri.TryCreate(baseUri, relativeUri, out var combined))
|
||||
{
|
||||
return combined;
|
||||
}
|
||||
|
||||
if (_resourceMap.ContainsKey(relativeUri))
|
||||
{
|
||||
return new Uri($"embedded:///{relativeUri}", UriKind.Absolute);
|
||||
}
|
||||
|
||||
return base.ResolveUri(baseUri, relativeUri);
|
||||
}
|
||||
|
||||
private string? ResolveResourceName(string? key)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (_resourceMap.TryGetValue(key, out var resource))
|
||||
{
|
||||
return resource;
|
||||
}
|
||||
|
||||
var fileName = Path.GetFileName(key);
|
||||
if (!string.IsNullOrEmpty(fileName) && _resourceMap.TryGetValue(fileName, out resource))
|
||||
{
|
||||
return resource;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
internal sealed class JvnSchemaValidationException : Exception
|
||||
{
|
||||
public JvnSchemaValidationException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public JvnSchemaValidationException(string message, Exception innerException)
|
||||
: base(message, innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
240
src/StellaOps.Concelier.Connector.Jvn/Internal/MyJvnClient.cs
Normal file
240
src/StellaOps.Concelier.Connector.Jvn/Internal/MyJvnClient.cs
Normal file
@@ -0,0 +1,240 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Jvn.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
|
||||
public sealed class MyJvnClient
|
||||
{
|
||||
private static readonly XNamespace RssNamespace = "http://purl.org/rss/1.0/";
|
||||
private static readonly XNamespace DcTermsNamespace = "http://purl.org/dc/terms/";
|
||||
private static readonly XNamespace SecNamespace = "http://jvn.jp/rss/mod_sec/3.0/";
|
||||
private static readonly XNamespace RdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
|
||||
private static readonly XNamespace StatusNamespace = "http://jvndb.jvn.jp/myjvn/Status";
|
||||
|
||||
private static readonly TimeSpan TokyoOffset = TimeSpan.FromHours(9);
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly JvnOptions _options;
|
||||
private readonly ILogger<MyJvnClient> _logger;
|
||||
|
||||
public MyJvnClient(IHttpClientFactory httpClientFactory, IOptions<JvnOptions> options, ILogger<MyJvnClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
internal async Task<IReadOnlyList<JvnOverviewItem>> GetOverviewAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken)
|
||||
{
|
||||
if (windowEnd <= windowStart)
|
||||
{
|
||||
throw new ArgumentException("windowEnd must be greater than windowStart", nameof(windowEnd));
|
||||
}
|
||||
|
||||
var items = new List<JvnOverviewItem>();
|
||||
var client = _httpClientFactory.CreateClient(JvnOptions.HttpClientName);
|
||||
|
||||
var startItem = 1;
|
||||
var pagesFetched = 0;
|
||||
|
||||
while (pagesFetched < _options.MaxOverviewPagesPerFetch)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var requestUri = BuildOverviewUri(windowStart, windowEnd, startItem);
|
||||
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var reader = XmlReader.Create(contentStream, new XmlReaderSettings { Async = true, IgnoreWhitespace = true, IgnoreComments = true });
|
||||
var document = await XDocument.LoadAsync(reader, LoadOptions.None, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var page = ParseOverviewPage(document);
|
||||
if (page.Items.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("JVN overview page starting at {StartItem} returned zero results", startItem);
|
||||
break;
|
||||
}
|
||||
|
||||
items.AddRange(page.Items);
|
||||
pagesFetched++;
|
||||
|
||||
if (page.ReturnedCount < _options.PageSize || startItem + _options.PageSize > page.TotalResults)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
startItem += _options.PageSize;
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private Uri BuildOverviewUri(DateTimeOffset windowStart, DateTimeOffset windowEnd, int startItem)
|
||||
{
|
||||
var (startYear, startMonth, startDay) = ToTokyoDateParts(windowStart);
|
||||
var (endYear, endMonth, endDay) = ToTokyoDateParts(windowEnd);
|
||||
|
||||
var parameters = new[]
|
||||
{
|
||||
new KeyValuePair<string, string>("method", "getVulnOverviewList"),
|
||||
new KeyValuePair<string, string>("feed", "hnd"),
|
||||
new KeyValuePair<string, string>("lang", "en"),
|
||||
new KeyValuePair<string, string>("rangeDatePublished", "n"),
|
||||
new KeyValuePair<string, string>("rangeDatePublic", "n"),
|
||||
new KeyValuePair<string, string>("rangeDateFirstPublished", "n"),
|
||||
new KeyValuePair<string, string>("dateFirstPublishedStartY", startYear),
|
||||
new KeyValuePair<string, string>("dateFirstPublishedStartM", startMonth),
|
||||
new KeyValuePair<string, string>("dateFirstPublishedStartD", startDay),
|
||||
new KeyValuePair<string, string>("dateFirstPublishedEndY", endYear),
|
||||
new KeyValuePair<string, string>("dateFirstPublishedEndM", endMonth),
|
||||
new KeyValuePair<string, string>("dateFirstPublishedEndD", endDay),
|
||||
new KeyValuePair<string, string>("startItem", startItem.ToString(CultureInfo.InvariantCulture)),
|
||||
new KeyValuePair<string, string>("maxCountItem", _options.PageSize.ToString(CultureInfo.InvariantCulture)),
|
||||
};
|
||||
|
||||
var query = BuildQueryString(parameters);
|
||||
|
||||
var builder = new UriBuilder(_options.BaseEndpoint)
|
||||
{
|
||||
Query = query,
|
||||
};
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private static (string Year, string Month, string Day) ToTokyoDateParts(DateTimeOffset timestamp)
|
||||
{
|
||||
var local = timestamp.ToOffset(TokyoOffset).Date;
|
||||
return (
|
||||
local.Year.ToString("D4", CultureInfo.InvariantCulture),
|
||||
local.Month.ToString("D2", CultureInfo.InvariantCulture),
|
||||
local.Day.ToString("D2", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
private static JvnOverviewPage ParseOverviewPage(XDocument document)
|
||||
{
|
||||
var items = new List<JvnOverviewItem>();
|
||||
|
||||
foreach (var item in document.Descendants(RssNamespace + "item"))
|
||||
{
|
||||
var identifier = item.Element(SecNamespace + "identifier")?.Value?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(identifier))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Uri? detailUri = null;
|
||||
var linkValue = item.Element(RssNamespace + "link")?.Value?.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(linkValue))
|
||||
{
|
||||
Uri.TryCreate(linkValue, UriKind.Absolute, out detailUri);
|
||||
}
|
||||
|
||||
if (detailUri is null)
|
||||
{
|
||||
var aboutValue = item.Attribute(RdfNamespace + "about")?.Value?.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(aboutValue))
|
||||
{
|
||||
Uri.TryCreate(aboutValue, UriKind.Absolute, out detailUri);
|
||||
}
|
||||
}
|
||||
|
||||
if (detailUri is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var title = item.Element(RssNamespace + "title")?.Value?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
title = identifier;
|
||||
}
|
||||
|
||||
var firstPublished = TryParseDate(item.Element(DcTermsNamespace + "issued")?.Value);
|
||||
var lastUpdated = TryParseDate(item.Element(DcTermsNamespace + "modified")?.Value);
|
||||
|
||||
items.Add(new JvnOverviewItem(identifier, detailUri, title!, firstPublished, lastUpdated));
|
||||
}
|
||||
|
||||
var statusElement = document.Root?.Element(StatusNamespace + "Status")
|
||||
?? document.Descendants(StatusNamespace + "Status").FirstOrDefault();
|
||||
|
||||
var totalResults = TryParseInt(statusElement?.Attribute("totalRes")?.Value) ?? items.Count;
|
||||
var returned = TryParseInt(statusElement?.Attribute("totalResRet")?.Value) ?? items.Count;
|
||||
var firstResult = TryParseInt(statusElement?.Attribute("firstRes")?.Value) ?? 1;
|
||||
|
||||
return new JvnOverviewPage(items, totalResults, returned, firstResult);
|
||||
}
|
||||
|
||||
private static DateTimeOffset? TryParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out var parsed)
|
||||
? parsed.ToUniversalTime()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static int? TryParseInt(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
internal Uri BuildDetailUri(string vulnerabilityId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(vulnerabilityId);
|
||||
|
||||
var query = BuildQueryString(new[]
|
||||
{
|
||||
new KeyValuePair<string, string>("method", "getVulnDetailInfo"),
|
||||
new KeyValuePair<string, string>("feed", "hnd"),
|
||||
new KeyValuePair<string, string>("lang", "en"),
|
||||
new KeyValuePair<string, string>("vulnId", vulnerabilityId.Trim()),
|
||||
});
|
||||
var builder = new UriBuilder(_options.BaseEndpoint)
|
||||
{
|
||||
Query = query,
|
||||
};
|
||||
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private static string BuildQueryString(IEnumerable<KeyValuePair<string, string>> parameters)
|
||||
{
|
||||
return string.Join(
|
||||
"&",
|
||||
parameters.Select(parameter =>
|
||||
$"{WebUtility.UrlEncode(parameter.Key)}={WebUtility.UrlEncode(parameter.Value)}"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user