Rename Concelier Source modules to Connector
This commit is contained in:
		
							
								
								
									
										38
									
								
								src/StellaOps.Concelier.Connector.Kisa/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										38
									
								
								src/StellaOps.Concelier.Connector.Kisa/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,38 @@ | ||||
| # AGENTS | ||||
| ## Role | ||||
| Deliver the KISA (Korea Internet & Security Agency) advisory connector to ingest Korean vulnerability alerts for Concelier’s regional coverage. | ||||
|  | ||||
| ## Scope | ||||
| - Identify KISA’s advisory feeds (RSS/Atom, JSON, HTML) and determine localisation requirements (Korean language parsing). | ||||
| - Implement fetch/cursor logic with retry/backoff, handling authentication if required. | ||||
| - Parse advisory content to extract summary, affected vendors/products, mitigation steps, CVEs, references. | ||||
| - Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (including vendor/language metadata). | ||||
| - Provide deterministic fixtures and regression tests. | ||||
|  | ||||
| ## Participants | ||||
| - `Source.Common` (HTTP/fetch utilities, DTO storage). | ||||
| - `Storage.Mongo` (raw/document/DTO/advisory stores, source state). | ||||
| - `Concelier.Models` (canonical data structures). | ||||
| - `Concelier.Testing` (integration fixtures and snapshots). | ||||
|  | ||||
| ## Interfaces & Contracts | ||||
| - Job kinds: `kisa:fetch`, `kisa:parse`, `kisa:map`. | ||||
| - Persist upstream caching metadata (e.g., ETag/Last-Modified) when available. | ||||
| - Alias set should include KISA advisory identifiers and CVE IDs. | ||||
|  | ||||
| ## In/Out of scope | ||||
| In scope: | ||||
| - Advisory ingestion, translation/normalisation, range primitives. | ||||
|  | ||||
| Out of scope: | ||||
| - Automated Korean↔English translations beyond summary normalization (unless required for canonical fields). | ||||
|  | ||||
| ## Observability & Security Expectations | ||||
| - Log fetch and mapping metrics; record failures with backoff. | ||||
| - Sanitise HTML, removing scripts/styles. | ||||
| - Handle character encoding (UTF-8/Korean) correctly. | ||||
|  | ||||
| ## Tests | ||||
| - Add `StellaOps.Concelier.Connector.Kisa.Tests` covering fetch/parse/map with Korean-language fixtures. | ||||
| - Snapshot canonical advisories; support fixture regeneration via env flag. | ||||
| - Ensure deterministic ordering/time normalisation. | ||||
| @@ -0,0 +1,97 @@ | ||||
| using System; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Configuration; | ||||
|  | ||||
| public sealed class KisaOptions | ||||
| { | ||||
|     public const string HttpClientName = "concelier.source.kisa"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Primary RSS feed for security advisories. | ||||
|     /// </summary> | ||||
|     public Uri FeedUri { get; set; } = new("https://knvd.krcert.or.kr/rss/securityInfo.do"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Detail API endpoint template; `IDX` query parameter identifies the advisory. | ||||
|     /// </summary> | ||||
|     public Uri DetailApiUri { get; set; } = new("https://knvd.krcert.or.kr/rssDetailData.do"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional HTML detail URI template for provenance. | ||||
|     /// </summary> | ||||
|     public Uri DetailPageUri { get; set; } = new("https://knvd.krcert.or.kr/detailDos.do"); | ||||
|  | ||||
|     public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); | ||||
|  | ||||
|     public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); | ||||
|  | ||||
|     public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     public int MaxAdvisoriesPerFetch { get; set; } = 20; | ||||
|  | ||||
|     public int MaxKnownAdvisories { get; set; } = 256; | ||||
|  | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (FeedUri is null || !FeedUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("KISA feed URI must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("KISA detail API URI must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (DetailPageUri is null || !DetailPageUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("KISA detail page URI must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (RequestTimeout <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("RequestTimeout must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (RequestDelay < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("RequestDelay cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (FailureBackoff <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("FailureBackoff must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (MaxAdvisoriesPerFetch <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("MaxAdvisoriesPerFetch must be greater than zero."); | ||||
|         } | ||||
|  | ||||
|         if (MaxKnownAdvisories <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("MaxKnownAdvisories must be greater than zero."); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public Uri BuildDetailApiUri(string idx) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(idx)) | ||||
|         { | ||||
|             throw new ArgumentException("IDX must not be empty", nameof(idx)); | ||||
|         } | ||||
|  | ||||
|         var builder = new UriBuilder(DetailApiUri); | ||||
|         var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; | ||||
|         builder.Query = $"{queryPrefix}IDX={Uri.EscapeDataString(idx)}"; | ||||
|         return builder.Uri; | ||||
|     } | ||||
|  | ||||
|     public Uri BuildDetailPageUri(string idx) | ||||
|     { | ||||
|         var builder = new UriBuilder(DetailPageUri); | ||||
|         var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; | ||||
|         builder.Query = $"{queryPrefix}IDX={Uri.EscapeDataString(idx)}"; | ||||
|         return builder.Uri; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										120
									
								
								src/StellaOps.Concelier.Connector.Kisa/Internal/KisaCursor.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										120
									
								
								src/StellaOps.Concelier.Connector.Kisa/Internal/KisaCursor.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,120 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| internal sealed record KisaCursor( | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings, | ||||
|     IReadOnlyCollection<string> KnownIds, | ||||
|     DateTimeOffset? LastPublished, | ||||
|     DateTimeOffset? LastFetchAt) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>(); | ||||
|     private static readonly IReadOnlyCollection<string> EmptyStrings = Array.Empty<string>(); | ||||
|  | ||||
|     public static KisaCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null); | ||||
|  | ||||
|     public KisaCursor WithPendingDocuments(IEnumerable<Guid> documents) | ||||
|         => this with { PendingDocuments = Distinct(documents) }; | ||||
|  | ||||
|     public KisaCursor WithPendingMappings(IEnumerable<Guid> mappings) | ||||
|         => this with { PendingMappings = Distinct(mappings) }; | ||||
|  | ||||
|     public KisaCursor WithKnownIds(IEnumerable<string> ids) | ||||
|         => this with { KnownIds = ids?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings }; | ||||
|  | ||||
|     public KisaCursor WithLastPublished(DateTimeOffset? published) | ||||
|         => this with { LastPublished = published }; | ||||
|  | ||||
|     public KisaCursor WithLastFetch(DateTimeOffset? timestamp) | ||||
|         => this with { LastFetchAt = timestamp }; | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), | ||||
|             ["knownIds"] = new BsonArray(KnownIds), | ||||
|         }; | ||||
|  | ||||
|         if (LastPublished.HasValue) | ||||
|         { | ||||
|             document["lastPublished"] = LastPublished.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (LastFetchAt.HasValue) | ||||
|         { | ||||
|             document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static KisaCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|         var knownIds = ReadStringArray(document, "knownIds"); | ||||
|         var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) | ||||
|             ? ParseDate(publishedValue) | ||||
|             : null; | ||||
|         var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue) | ||||
|             ? ParseDate(fetchValue) | ||||
|             : null; | ||||
|  | ||||
|         return new KisaCursor(pendingDocuments, pendingMappings, knownIds, lastPublished, lastFetch); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values) | ||||
|         => values?.Distinct().ToArray() ?? EmptyGuids; | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuids; | ||||
|         } | ||||
|  | ||||
|         var items = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element?.ToString(), out var id)) | ||||
|             { | ||||
|                 items.Add(id); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return items; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyStrings; | ||||
|         } | ||||
|  | ||||
|         return array | ||||
|             .Select(element => element?.ToString() ?? string.Empty) | ||||
|             .Where(static s => !string.IsNullOrWhiteSpace(s)) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(BsonValue value) | ||||
|         => value.BsonType switch | ||||
|         { | ||||
|             BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), | ||||
|             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|             _ => null, | ||||
|         }; | ||||
| } | ||||
| @@ -0,0 +1,114 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using System.Text.Json.Serialization; | ||||
| using StellaOps.Concelier.Connector.Common.Html; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| public sealed class KisaDetailParser | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, | ||||
|     }; | ||||
|  | ||||
|     private readonly HtmlContentSanitizer _sanitizer; | ||||
|  | ||||
|     public KisaDetailParser(HtmlContentSanitizer sanitizer) | ||||
|         => _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); | ||||
|  | ||||
|     public KisaParsedAdvisory Parse(Uri detailApiUri, Uri detailPageUri, byte[] payload) | ||||
|     { | ||||
|         var response = JsonSerializer.Deserialize<KisaDetailResponse>(payload, SerializerOptions) | ||||
|             ?? throw new InvalidOperationException("KISA detail payload deserialized to null"); | ||||
|  | ||||
|         var idx = response.Idx ?? throw new InvalidOperationException("KISA detail missing IDX"); | ||||
|         var contentHtml = _sanitizer.Sanitize(response.ContentHtml ?? string.Empty, detailPageUri); | ||||
|  | ||||
|         return new KisaParsedAdvisory( | ||||
|             idx, | ||||
|             Normalize(response.Title) ?? idx, | ||||
|             Normalize(response.Summary), | ||||
|             contentHtml, | ||||
|             Normalize(response.Severity), | ||||
|             response.Published, | ||||
|             response.Updated ?? response.Published, | ||||
|             detailApiUri, | ||||
|             detailPageUri, | ||||
|             NormalizeArray(response.CveIds), | ||||
|             MapReferences(response.References), | ||||
|             MapProducts(response.Products)); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> NormalizeArray(string[]? values) | ||||
|     { | ||||
|         if (values is null || values.Length == 0) | ||||
|         { | ||||
|             return Array.Empty<string>(); | ||||
|         } | ||||
|  | ||||
|         return values | ||||
|             .Select(Normalize) | ||||
|             .Where(static value => !string.IsNullOrWhiteSpace(value)) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray()!; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<KisaParsedReference> MapReferences(KisaReferenceDto[]? references) | ||||
|     { | ||||
|         if (references is null || references.Length == 0) | ||||
|         { | ||||
|             return Array.Empty<KisaParsedReference>(); | ||||
|         } | ||||
|  | ||||
|         return references | ||||
|             .Where(static reference => !string.IsNullOrWhiteSpace(reference.Url)) | ||||
|             .Select(reference => new KisaParsedReference(reference.Url!, Normalize(reference.Label))) | ||||
|             .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<KisaParsedProduct> MapProducts(KisaProductDto[]? products) | ||||
|     { | ||||
|         if (products is null || products.Length == 0) | ||||
|         { | ||||
|             return Array.Empty<KisaParsedProduct>(); | ||||
|         } | ||||
|  | ||||
|         return products | ||||
|             .Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name)) | ||||
|             .Select(product => new KisaParsedProduct( | ||||
|                 Normalize(product.Vendor), | ||||
|                 Normalize(product.Name), | ||||
|                 Normalize(product.Versions))) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static string? Normalize(string? value) | ||||
|         => string.IsNullOrWhiteSpace(value) | ||||
|             ? null | ||||
|             : value.Normalize(NormalizationForm.FormC).Trim(); | ||||
| } | ||||
|  | ||||
| public sealed record KisaParsedAdvisory( | ||||
|     string AdvisoryId, | ||||
|     string Title, | ||||
|     string? Summary, | ||||
|     string ContentHtml, | ||||
|     string? Severity, | ||||
|     DateTimeOffset? Published, | ||||
|     DateTimeOffset? Modified, | ||||
|     Uri DetailApiUri, | ||||
|     Uri DetailPageUri, | ||||
|     IReadOnlyList<string> CveIds, | ||||
|     IReadOnlyList<KisaParsedReference> References, | ||||
|     IReadOnlyList<KisaParsedProduct> Products); | ||||
|  | ||||
| public sealed record KisaParsedReference(string Url, string? Label); | ||||
|  | ||||
| public sealed record KisaParsedProduct(string? Vendor, string? Name, string? Versions); | ||||
| @@ -0,0 +1,58 @@ | ||||
| using System; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| internal sealed class KisaDetailResponse | ||||
| { | ||||
|     [JsonPropertyName("idx")] | ||||
|     public string? Idx { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("title")] | ||||
|     public string? Title { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("summary")] | ||||
|     public string? Summary { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("contentHtml")] | ||||
|     public string? ContentHtml { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("severity")] | ||||
|     public string? Severity { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("published")] | ||||
|     public DateTimeOffset? Published { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("updated")] | ||||
|     public DateTimeOffset? Updated { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cveIds")] | ||||
|     public string[]? CveIds { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("references")] | ||||
|     public KisaReferenceDto[]? References { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("products")] | ||||
|     public KisaProductDto[]? Products { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed class KisaReferenceDto | ||||
| { | ||||
|     [JsonPropertyName("url")] | ||||
|     public string? Url { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("label")] | ||||
|     public string? Label { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed class KisaProductDto | ||||
| { | ||||
|     [JsonPropertyName("vendor")] | ||||
|     public string? Vendor { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("name")] | ||||
|     public string? Name { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("versions")] | ||||
|     public string? Versions { get; init; } | ||||
| } | ||||
| @@ -0,0 +1,169 @@ | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| public sealed class KisaDiagnostics : IDisposable | ||||
| { | ||||
|     public const string MeterName = "StellaOps.Concelier.Connector.Kisa"; | ||||
|     private const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _feedAttempts; | ||||
|     private readonly Counter<long> _feedSuccess; | ||||
|     private readonly Counter<long> _feedFailures; | ||||
|     private readonly Counter<long> _feedItems; | ||||
|     private readonly Counter<long> _detailAttempts; | ||||
|     private readonly Counter<long> _detailSuccess; | ||||
|     private readonly Counter<long> _detailUnchanged; | ||||
|     private readonly Counter<long> _detailFailures; | ||||
|     private readonly Counter<long> _parseAttempts; | ||||
|     private readonly Counter<long> _parseSuccess; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Counter<long> _mapSuccess; | ||||
|     private readonly Counter<long> _mapFailures; | ||||
|     private readonly Counter<long> _cursorUpdates; | ||||
|  | ||||
|     public KisaDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _feedAttempts = _meter.CreateCounter<long>( | ||||
|             name: "kisa.feed.attempts", | ||||
|             unit: "operations", | ||||
|             description: "Number of RSS fetch attempts performed for the KISA connector."); | ||||
|         _feedSuccess = _meter.CreateCounter<long>( | ||||
|             name: "kisa.feed.success", | ||||
|             unit: "operations", | ||||
|             description: "Number of RSS fetch attempts that completed successfully."); | ||||
|         _feedFailures = _meter.CreateCounter<long>( | ||||
|             name: "kisa.feed.failures", | ||||
|             unit: "operations", | ||||
|             description: "Number of RSS fetch attempts that failed."); | ||||
|         _feedItems = _meter.CreateCounter<long>( | ||||
|             name: "kisa.feed.items", | ||||
|             unit: "items", | ||||
|             description: "Number of feed items returned by successful RSS fetches."); | ||||
|         _detailAttempts = _meter.CreateCounter<long>( | ||||
|             name: "kisa.detail.attempts", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory detail fetch attempts."); | ||||
|         _detailSuccess = _meter.CreateCounter<long>( | ||||
|             name: "kisa.detail.success", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory detail documents fetched successfully."); | ||||
|         _detailUnchanged = _meter.CreateCounter<long>( | ||||
|             name: "kisa.detail.unchanged", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory detail fetches that returned HTTP 304 (no change)."); | ||||
|         _detailFailures = _meter.CreateCounter<long>( | ||||
|             name: "kisa.detail.failures", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory detail fetch attempts that failed."); | ||||
|         _parseAttempts = _meter.CreateCounter<long>( | ||||
|             name: "kisa.parse.attempts", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory documents queued for parsing."); | ||||
|         _parseSuccess = _meter.CreateCounter<long>( | ||||
|             name: "kisa.parse.success", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory documents parsed successfully into DTOs."); | ||||
|         _parseFailures = _meter.CreateCounter<long>( | ||||
|             name: "kisa.parse.failures", | ||||
|             unit: "documents", | ||||
|             description: "Number of advisory documents that failed parsing."); | ||||
|         _mapSuccess = _meter.CreateCounter<long>( | ||||
|             name: "kisa.map.success", | ||||
|             unit: "advisories", | ||||
|             description: "Number of canonical advisories produced by the mapper."); | ||||
|         _mapFailures = _meter.CreateCounter<long>( | ||||
|             name: "kisa.map.failures", | ||||
|             unit: "advisories", | ||||
|             description: "Number of advisories that failed to map."); | ||||
|         _cursorUpdates = _meter.CreateCounter<long>( | ||||
|             name: "kisa.cursor.updates", | ||||
|             unit: "updates", | ||||
|             description: "Number of times the published cursor advanced."); | ||||
|     } | ||||
|  | ||||
|     public void FeedAttempt() => _feedAttempts.Add(1); | ||||
|  | ||||
|     public void FeedSuccess(int itemCount) | ||||
|     { | ||||
|         _feedSuccess.Add(1); | ||||
|         if (itemCount > 0) | ||||
|         { | ||||
|             _feedItems.Add(itemCount); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void FeedFailure(string reason) | ||||
|         => _feedFailures.Add(1, GetReasonTags(reason)); | ||||
|  | ||||
|     public void DetailAttempt(string? category) | ||||
|         => _detailAttempts.Add(1, GetCategoryTags(category)); | ||||
|  | ||||
|     public void DetailSuccess(string? category) | ||||
|         => _detailSuccess.Add(1, GetCategoryTags(category)); | ||||
|  | ||||
|     public void DetailUnchanged(string? category) | ||||
|         => _detailUnchanged.Add(1, GetCategoryTags(category)); | ||||
|  | ||||
|     public void DetailFailure(string? category, string reason) | ||||
|         => _detailFailures.Add(1, GetCategoryReasonTags(category, reason)); | ||||
|  | ||||
|     public void ParseAttempt(string? category) | ||||
|         => _parseAttempts.Add(1, GetCategoryTags(category)); | ||||
|  | ||||
|     public void ParseSuccess(string? category) | ||||
|         => _parseSuccess.Add(1, GetCategoryTags(category)); | ||||
|  | ||||
|     public void ParseFailure(string? category, string reason) | ||||
|         => _parseFailures.Add(1, GetCategoryReasonTags(category, reason)); | ||||
|  | ||||
|     public void MapSuccess(string? severity) | ||||
|         => _mapSuccess.Add(1, GetSeverityTags(severity)); | ||||
|  | ||||
|     public void MapFailure(string? severity, string reason) | ||||
|         => _mapFailures.Add(1, GetSeverityReasonTags(severity, reason)); | ||||
|  | ||||
|     public void CursorAdvanced() | ||||
|         => _cursorUpdates.Add(1); | ||||
|  | ||||
|     public Meter Meter => _meter; | ||||
|  | ||||
|     public void Dispose() => _meter.Dispose(); | ||||
|  | ||||
|     private static KeyValuePair<string, object?>[] GetCategoryTags(string? category) | ||||
|         => new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("category", Normalize(category)) | ||||
|         }; | ||||
|  | ||||
|     private static KeyValuePair<string, object?>[] GetCategoryReasonTags(string? category, string reason) | ||||
|         => new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("category", Normalize(category)), | ||||
|             new KeyValuePair<string, object?>("reason", Normalize(reason)), | ||||
|         }; | ||||
|  | ||||
|     private static KeyValuePair<string, object?>[] GetSeverityTags(string? severity) | ||||
|         => new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("severity", Normalize(severity)), | ||||
|         }; | ||||
|  | ||||
|     private static KeyValuePair<string, object?>[] GetSeverityReasonTags(string? severity, string reason) | ||||
|         => new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("severity", Normalize(severity)), | ||||
|             new KeyValuePair<string, object?>("reason", Normalize(reason)), | ||||
|         }; | ||||
|  | ||||
|     private static KeyValuePair<string, object?>[] GetReasonTags(string reason) | ||||
|         => new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("reason", Normalize(reason)), | ||||
|         }; | ||||
|  | ||||
|     private static string Normalize(string? value) | ||||
|         => string.IsNullOrWhiteSpace(value) ? "unknown" : value!; | ||||
| } | ||||
| @@ -0,0 +1,29 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| internal static class KisaDocumentMetadata | ||||
| { | ||||
|     public static Dictionary<string, string> CreateMetadata(KisaFeedItem item) | ||||
|     { | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             ["kisa.idx"] = item.AdvisoryId, | ||||
|             ["kisa.detailPage"] = item.DetailPageUri.ToString(), | ||||
|             ["kisa.published"] = item.Published.ToString("O"), | ||||
|         }; | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(item.Title)) | ||||
|         { | ||||
|             metadata["kisa.title"] = item.Title!; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(item.Category)) | ||||
|         { | ||||
|             metadata["kisa.category"] = item.Category!; | ||||
|         } | ||||
|  | ||||
|         return metadata; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,116 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using System.Net.Http; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using System.Xml.Linq; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Concelier.Connector.Kisa.Configuration; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| public sealed class KisaFeedClient | ||||
| { | ||||
|     private readonly IHttpClientFactory _httpClientFactory; | ||||
|     private readonly KisaOptions _options; | ||||
|     private readonly ILogger<KisaFeedClient> _logger; | ||||
|  | ||||
|     public KisaFeedClient( | ||||
|         IHttpClientFactory httpClientFactory, | ||||
|         IOptions<KisaOptions> options, | ||||
|         ILogger<KisaFeedClient> logger) | ||||
|     { | ||||
|         _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<IReadOnlyList<KisaFeedItem>> LoadAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var client = _httpClientFactory.CreateClient(KisaOptions.HttpClientName); | ||||
|  | ||||
|         using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri); | ||||
|         request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8"); | ||||
|         using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); | ||||
|         response.EnsureSuccessStatusCode(); | ||||
|  | ||||
|         await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var document = XDocument.Load(stream); | ||||
|  | ||||
|         var items = new List<KisaFeedItem>(); | ||||
|         foreach (var element in document.Descendants("item")) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var link = element.Element("link")?.Value?.Trim(); | ||||
|             if (string.IsNullOrWhiteSpace(link)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!TryExtractIdx(link, out var idx)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var title = element.Element("title")?.Value?.Trim(); | ||||
|             var category = element.Element("category")?.Value?.Trim(); | ||||
|             var published = ParseDate(element.Element("pubDate")?.Value); | ||||
|             var detailApiUri = _options.BuildDetailApiUri(idx); | ||||
|             var detailPageUri = _options.BuildDetailPageUri(idx); | ||||
|  | ||||
|             items.Add(new KisaFeedItem(idx, detailApiUri, detailPageUri, published, title, category)); | ||||
|         } | ||||
|  | ||||
|         return items; | ||||
|     } | ||||
|  | ||||
|     private static bool TryExtractIdx(string link, out string idx) | ||||
|     { | ||||
|         idx = string.Empty; | ||||
|         if (string.IsNullOrWhiteSpace(link)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (!Uri.TryCreate(link, UriKind.Absolute, out var uri)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         var query = uri.Query?.TrimStart('?'); | ||||
|         if (string.IsNullOrEmpty(query)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) | ||||
|         { | ||||
|             var separatorIndex = pair.IndexOf('='); | ||||
|             if (separatorIndex <= 0) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var key = pair[..separatorIndex].Trim(); | ||||
|             if (!key.Equals("IDX", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             idx = Uri.UnescapeDataString(pair[(separatorIndex + 1)..]); | ||||
|             return !string.IsNullOrWhiteSpace(idx); | ||||
|         } | ||||
|  | ||||
|         return false; | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset ParseDate(string? value) | ||||
|         => DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) | ||||
|             ? parsed | ||||
|             : DateTimeOffset.UtcNow; | ||||
| } | ||||
| @@ -0,0 +1,11 @@ | ||||
| using System; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| public sealed record KisaFeedItem( | ||||
|     string AdvisoryId, | ||||
|     Uri DetailApiUri, | ||||
|     Uri DetailPageUri, | ||||
|     DateTimeOffset Published, | ||||
|     string? Title, | ||||
|     string? Category); | ||||
							
								
								
									
										145
									
								
								src/StellaOps.Concelier.Connector.Kisa/Internal/KisaMapper.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										145
									
								
								src/StellaOps.Concelier.Connector.Kisa/Internal/KisaMapper.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,145 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| internal static class KisaMapper | ||||
| { | ||||
|     public static Advisory Map(KisaParsedAdvisory dto, DocumentRecord document, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         var aliases = BuildAliases(dto); | ||||
|         var references = BuildReferences(dto, recordedAt); | ||||
|         var packages = BuildPackages(dto, recordedAt); | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             KisaConnectorPlugin.SourceName, | ||||
|             "advisory", | ||||
|             dto.AdvisoryId, | ||||
|             recordedAt, | ||||
|             new[] { ProvenanceFieldMasks.Advisory }); | ||||
|  | ||||
|         return new Advisory( | ||||
|             advisoryKey: dto.AdvisoryId, | ||||
|             title: dto.Title, | ||||
|             summary: dto.Summary, | ||||
|             language: "ko", | ||||
|             published: dto.Published, | ||||
|             modified: dto.Modified, | ||||
|             severity: dto.Severity?.ToLowerInvariant(), | ||||
|             exploitKnown: false, | ||||
|             aliases: aliases, | ||||
|             references: references, | ||||
|             affectedPackages: packages, | ||||
|             cvssMetrics: Array.Empty<CvssMetric>(), | ||||
|             provenance: new[] { provenance }); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> BuildAliases(KisaParsedAdvisory dto) | ||||
|     { | ||||
|         var aliases = new List<string>(capacity: dto.CveIds.Count + 1) { dto.AdvisoryId }; | ||||
|         aliases.AddRange(dto.CveIds); | ||||
|         return aliases | ||||
|             .Where(static alias => !string.IsNullOrWhiteSpace(alias)) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryReference> BuildReferences(KisaParsedAdvisory dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var references = new List<AdvisoryReference> | ||||
|         { | ||||
|             new(dto.DetailPageUri.ToString(), "details", "kisa", null, new AdvisoryProvenance( | ||||
|                 KisaConnectorPlugin.SourceName, | ||||
|                 "reference", | ||||
|                 dto.DetailPageUri.ToString(), | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.References })) | ||||
|         }; | ||||
|  | ||||
|         foreach (var reference in dto.References) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(reference.Url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             references.Add(new AdvisoryReference( | ||||
|                 reference.Url, | ||||
|                 kind: "reference", | ||||
|                 sourceTag: "kisa", | ||||
|                 summary: reference.Label, | ||||
|                 provenance: new AdvisoryProvenance( | ||||
|                     KisaConnectorPlugin.SourceName, | ||||
|                     "reference", | ||||
|                     reference.Url, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.References }))); | ||||
|         } | ||||
|  | ||||
|         return references | ||||
|             .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> BuildPackages(KisaParsedAdvisory dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Products.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Products.Count); | ||||
|         foreach (var product in dto.Products) | ||||
|         { | ||||
|             var vendor = string.IsNullOrWhiteSpace(product.Vendor) ? "Unknown" : product.Vendor!; | ||||
|             var name = product.Name; | ||||
|             var identifier = string.IsNullOrWhiteSpace(name) ? vendor : $"{vendor} {name}"; | ||||
|  | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 KisaConnectorPlugin.SourceName, | ||||
|                 "package", | ||||
|                 identifier, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.AffectedPackages }); | ||||
|  | ||||
|             var versionRanges = string.IsNullOrWhiteSpace(product.Versions) | ||||
|                 ? Array.Empty<AffectedVersionRange>() | ||||
|                 : new[] | ||||
|                     { | ||||
|                         new AffectedVersionRange( | ||||
|                             rangeKind: "string", | ||||
|                             introducedVersion: null, | ||||
|                             fixedVersion: null, | ||||
|                             lastAffectedVersion: null, | ||||
|                             rangeExpression: product.Versions, | ||||
|                             provenance: new AdvisoryProvenance( | ||||
|                                 KisaConnectorPlugin.SourceName, | ||||
|                                 "package-range", | ||||
|                                 product.Versions, | ||||
|                                 recordedAt, | ||||
|                                 new[] { ProvenanceFieldMasks.VersionRanges })) | ||||
|                     }; | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 AffectedPackageTypes.Vendor, | ||||
|                 identifier, | ||||
|                 platform: null, | ||||
|                 versionRanges: versionRanges, | ||||
|                 statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                 provenance: new[] { provenance }, | ||||
|                 normalizedVersions: Array.Empty<NormalizedVersionRule>())); | ||||
|         } | ||||
|  | ||||
|         return packages | ||||
|             .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										22
									
								
								src/StellaOps.Concelier.Connector.Kisa/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										22
									
								
								src/StellaOps.Concelier.Connector.Kisa/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,22 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Concelier.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa; | ||||
|  | ||||
| internal static class KisaJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:kisa:fetch"; | ||||
| } | ||||
|  | ||||
| internal sealed class KisaFetchJob : IJob | ||||
| { | ||||
|     private readonly KisaConnector _connector; | ||||
|  | ||||
|     public KisaFetchJob(KisaConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
							
								
								
									
										404
									
								
								src/StellaOps.Concelier.Connector.Kisa/KisaConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										404
									
								
								src/StellaOps.Concelier.Connector.Kisa/KisaConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,404 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Concelier.Connector.Common; | ||||
| using StellaOps.Concelier.Connector.Common.Fetch; | ||||
| using StellaOps.Concelier.Connector.Kisa.Configuration; | ||||
| using StellaOps.Concelier.Connector.Kisa.Internal; | ||||
| using StellaOps.Concelier.Storage.Mongo; | ||||
| using StellaOps.Concelier.Storage.Mongo.Advisories; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
| using StellaOps.Concelier.Storage.Mongo.Dtos; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa; | ||||
|  | ||||
| public sealed class KisaConnector : IFeedConnector | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, | ||||
|     }; | ||||
|  | ||||
|     private readonly KisaFeedClient _feedClient; | ||||
|     private readonly KisaDetailParser _detailParser; | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly KisaOptions _options; | ||||
|     private readonly KisaDiagnostics _diagnostics; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<KisaConnector> _logger; | ||||
|  | ||||
|     public KisaConnector( | ||||
|         KisaFeedClient feedClient, | ||||
|         KisaDetailParser detailParser, | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         IOptions<KisaOptions> options, | ||||
|         KisaDiagnostics diagnostics, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<KisaConnector> logger) | ||||
|     { | ||||
|         _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); | ||||
|         _detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser)); | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => KisaConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         _diagnostics.FeedAttempt(); | ||||
|         IReadOnlyList<KisaFeedItem> items; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             items = await _feedClient.LoadAsync(cancellationToken).ConfigureAwait(false); | ||||
|             _diagnostics.FeedSuccess(items.Count); | ||||
|  | ||||
|             if (items.Count > 0) | ||||
|             { | ||||
|                 _logger.LogInformation("KISA feed returned {ItemCount} advisories", items.Count); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 _logger.LogDebug("KISA feed returned no advisories"); | ||||
|             } | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _diagnostics.FeedFailure(ex.GetType().Name); | ||||
|             _logger.LogError(ex, "KISA feed fetch failed"); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|  | ||||
|         if (items.Count == 0) | ||||
|         { | ||||
|             await UpdateCursorAsync(cursor.WithLastFetch(now), cancellationToken).ConfigureAwait(false); | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|         var knownIds = new HashSet<string>(cursor.KnownIds, StringComparer.OrdinalIgnoreCase); | ||||
|         var processed = 0; | ||||
|         var latestPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; | ||||
|  | ||||
|         foreach (var item in items.OrderByDescending(static i => i.Published)) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             if (knownIds.Contains(item.AdvisoryId)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (processed >= _options.MaxAdvisoriesPerFetch) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             var category = item.Category; | ||||
|             _diagnostics.DetailAttempt(category); | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailApiUri.ToString(), cancellationToken).ConfigureAwait(false); | ||||
|                 var request = new SourceFetchRequest(KisaOptions.HttpClientName, SourceName, item.DetailApiUri) | ||||
|                 { | ||||
|                     Metadata = KisaDocumentMetadata.CreateMetadata(item), | ||||
|                     AcceptHeaders = new[] { "application/json", "text/json" }, | ||||
|                     ETag = existing?.Etag, | ||||
|                     LastModified = existing?.LastModified, | ||||
|                     TimeoutOverride = _options.RequestTimeout, | ||||
|                 }; | ||||
|  | ||||
|                 var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|                 if (result.IsNotModified) | ||||
|                 { | ||||
|                     _diagnostics.DetailUnchanged(category); | ||||
|                     _logger.LogDebug("KISA detail {Idx} unchanged ({Category})", item.AdvisoryId, category ?? "unknown"); | ||||
|                     knownIds.Add(item.AdvisoryId); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!result.IsSuccess || result.Document is null) | ||||
|                 { | ||||
|                     _diagnostics.DetailFailure(category, "empty-document"); | ||||
|                     _logger.LogWarning("KISA detail fetch returned no document for {Idx}", item.AdvisoryId); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 pendingDocuments.Add(result.Document.Id); | ||||
|                 pendingMappings.Remove(result.Document.Id); | ||||
|                 knownIds.Add(item.AdvisoryId); | ||||
|                 processed++; | ||||
|                 _diagnostics.DetailSuccess(category); | ||||
|                 _logger.LogInformation( | ||||
|                     "KISA fetched detail for {Idx} (documentId={DocumentId}, category={Category})", | ||||
|                     item.AdvisoryId, | ||||
|                     result.Document.Id, | ||||
|                     category ?? "unknown"); | ||||
|  | ||||
|                 if (_options.RequestDelay > TimeSpan.Zero) | ||||
|                 { | ||||
|                     await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.DetailFailure(category, ex.GetType().Name); | ||||
|                 _logger.LogError(ex, "KISA detail fetch failed for {Idx}", item.AdvisoryId); | ||||
|                 await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             if (item.Published > latestPublished) | ||||
|             { | ||||
|                 latestPublished = item.Published; | ||||
|                 _diagnostics.CursorAdvanced(); | ||||
|                 _logger.LogDebug("KISA advanced published cursor to {Published:O}", latestPublished); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var trimmedKnown = knownIds.Count > _options.MaxKnownAdvisories | ||||
|             ? knownIds.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase) | ||||
|                 .Take(_options.MaxKnownAdvisories) | ||||
|                 .ToArray() | ||||
|             : knownIds.ToArray(); | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings) | ||||
|             .WithKnownIds(trimmedKnown) | ||||
|             .WithLastPublished(latestPublished == DateTimeOffset.MinValue ? cursor.LastPublished : latestPublished) | ||||
|             .WithLastFetch(now); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|         _logger.LogInformation("KISA fetch stored {Processed} new documents (knownIds={KnownCount})", processed, trimmedKnown.Length); | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remainingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(null, "document-missing"); | ||||
|                 _logger.LogWarning("KISA document {DocumentId} missing during parse", documentId); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var category = GetCategory(document); | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(category, "missing-gridfs"); | ||||
|                 _logger.LogWarning("KISA document {DocumentId} missing GridFS payload", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             _diagnostics.ParseAttempt(category); | ||||
|  | ||||
|             byte[] payload; | ||||
|             try | ||||
|             { | ||||
|                 payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(category, "download"); | ||||
|                 _logger.LogError(ex, "KISA unable to download document {DocumentId}", document.Id); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             KisaParsedAdvisory parsed; | ||||
|             try | ||||
|             { | ||||
|                 var apiUri = new Uri(document.Uri); | ||||
|                 var pageUri = document.Metadata is not null && document.Metadata.TryGetValue("kisa.detailPage", out var pageValue) | ||||
|                     ? new Uri(pageValue) | ||||
|                     : apiUri; | ||||
|                 parsed = _detailParser.Parse(apiUri, pageUri, payload); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(category, "parse"); | ||||
|                 _logger.LogError(ex, "KISA failed to parse detail {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|             _diagnostics.ParseSuccess(category); | ||||
|             _logger.LogDebug("KISA parsed detail for {DocumentId} ({Category})", document.Id, category ?? "unknown"); | ||||
|  | ||||
|             var dtoBson = BsonDocument.Parse(JsonSerializer.Serialize(parsed, SerializerOptions)); | ||||
|             var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "kisa.detail.v1", dtoBson, now); | ||||
|             await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             remainingDocuments.Remove(documentId); | ||||
|             pendingMappings.Add(document.Id); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(remainingDocuments) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(null, "document-missing"); | ||||
|                 _logger.LogWarning("KISA document {DocumentId} missing during map", documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (dtoRecord is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(null, "dto-missing"); | ||||
|                 _logger.LogWarning("KISA DTO missing for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             KisaParsedAdvisory? parsed; | ||||
|             try | ||||
|             { | ||||
|                 parsed = JsonSerializer.Deserialize<KisaParsedAdvisory>(dtoRecord.Payload.ToJson(), SerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(null, "dto-deserialize"); | ||||
|                 _logger.LogError(ex, "KISA failed to deserialize DTO for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (parsed is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(null, "dto-null"); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var advisory = KisaMapper.Map(parsed, document, dtoRecord.ValidatedAt); | ||||
|                 await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 _diagnostics.MapSuccess(parsed.Severity); | ||||
|                 _logger.LogInformation("KISA mapped advisory {AdvisoryId} (severity={Severity})", parsed.AdvisoryId, parsed.Severity ?? "unknown"); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(parsed.Severity, "map"); | ||||
|                 _logger.LogError(ex, "KISA mapping failed for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private static string? GetCategory(DocumentRecord document) | ||||
|     { | ||||
|         if (document.Metadata is null) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return document.Metadata.TryGetValue("kisa.category", out var category) | ||||
|             ? category | ||||
|             : null; | ||||
|     } | ||||
|  | ||||
|     private async Task<KisaCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? KisaCursor.Empty : KisaCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private Task UpdateCursorAsync(KisaCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var document = cursor.ToBsonDocument(); | ||||
|         var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow(); | ||||
|         return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,21 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa; | ||||
|  | ||||
| public sealed class KisaConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public const string SourceName = "kisa"; | ||||
|  | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) | ||||
|         => services.GetService<KisaConnector>() is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return services.GetRequiredService<KisaConnector>(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,50 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Concelier.Core.Jobs; | ||||
| using StellaOps.Concelier.Connector.Kisa.Configuration; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa; | ||||
|  | ||||
| public sealed class KisaDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "concelier:sources:kisa"; | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddKisaConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         services.AddTransient<KisaFetchJob>(); | ||||
|  | ||||
|         services.PostConfigure<JobSchedulerOptions>(options => | ||||
|         { | ||||
|             EnsureJob(options, KisaJobKinds.Fetch, typeof(KisaFetchJob)); | ||||
|         }); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
|  | ||||
|     private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) | ||||
|     { | ||||
|         if (options.Definitions.ContainsKey(kind)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         options.Definitions[kind] = new JobDefinition( | ||||
|             kind, | ||||
|             jobType, | ||||
|             options.DefaultTimeout, | ||||
|             options.DefaultLeaseDuration, | ||||
|             CronExpression: null, | ||||
|             Enabled: true); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,47 @@ | ||||
| using System; | ||||
| using System.Net; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.DependencyInjection.Extensions; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Concelier.Connector.Common.Html; | ||||
| using StellaOps.Concelier.Connector.Common.Http; | ||||
| using StellaOps.Concelier.Connector.Kisa.Configuration; | ||||
| using StellaOps.Concelier.Connector.Kisa.Internal; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Kisa; | ||||
|  | ||||
| public static class KisaServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddKisaConnector(this IServiceCollection services, Action<KisaOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<KisaOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static options => options.Validate()); | ||||
|  | ||||
|         services.AddSourceHttpClient(KisaOptions.HttpClientName, static (sp, clientOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<KisaOptions>>().Value; | ||||
|             clientOptions.Timeout = options.RequestTimeout; | ||||
|             clientOptions.UserAgent = "StellaOps.Concelier.Kisa/1.0"; | ||||
|             clientOptions.DefaultRequestHeaders["Accept-Language"] = "ko-KR"; | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add(options.FeedUri.Host); | ||||
|             clientOptions.AllowedHosts.Add(options.DetailApiUri.Host); | ||||
|             clientOptions.ConfigureHandler = handler => | ||||
|             { | ||||
|                 handler.AutomaticDecompression = DecompressionMethods.All; | ||||
|                 handler.AllowAutoRedirect = true; | ||||
|             }; | ||||
|         }); | ||||
|  | ||||
|         services.TryAddSingleton<HtmlContentSanitizer>(); | ||||
|         services.TryAddSingleton<KisaFeedClient>(); | ||||
|         services.TryAddSingleton<KisaDetailParser>(); | ||||
|         services.AddSingleton<KisaDiagnostics>(); | ||||
|         services.AddTransient<KisaConnector>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,17 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|  | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|  | ||||
							
								
								
									
										10
									
								
								src/StellaOps.Concelier.Connector.Kisa/TASKS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										10
									
								
								src/StellaOps.Concelier.Connector.Kisa/TASKS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,10 @@ | ||||
| # TASKS | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |FEEDCONN-KISA-02-001 Research KISA advisory feeds|BE-Conn-KISA|Research|**DONE (2025-10-11)** – Located public RSS endpoints (`https://knvd.krcert.or.kr/rss/securityInfo.do`, `.../securityNotice.do`) returning UTF-8 XML with 10-item windows and canonical `detailDos.do?IDX=` links. Logged output structure + header profile in `docs/concelier-connector-research-20251011.md`; outstanding work is parsing the SPA detail payload.| | ||||
| |FEEDCONN-KISA-02-002 Fetch pipeline & source state|BE-Conn-KISA|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `KisaConnector.FetchAsync` pulls RSS, sets `Accept-Language: ko-KR`, persists detail JSON with IDX metadata, throttles requests, and tracks cursor state (pending docs/mappings, known IDs, published timestamp).| | ||||
| |FEEDCONN-KISA-02-003 Parser & DTO implementation|BE-Conn-KISA|Source.Common|**DONE (2025-10-14)** – Detail API parsed via `KisaDetailParser` (Hangul NFC normalisation, sanitised HTML, CVE extraction, references/products captured into DTO `kisa.detail.v1`).| | ||||
| |FEEDCONN-KISA-02-004 Canonical mapping & range primitives|BE-Conn-KISA|Models|**DONE (2025-10-14)** – `KisaMapper` emits vendor packages with range strings, aliases (IDX/CVEs), references, and provenance; advisories default to `ko` language and normalised severity.| | ||||
| |FEEDCONN-KISA-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added `StellaOps.Concelier.Connector.Kisa.Tests` with Korean fixtures and fetch→parse→map regression; fixtures regenerate via `UPDATE_KISA_FIXTURES=1`.| | ||||
| |FEEDCONN-KISA-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Added diagnostics-backed telemetry, structured logs, regression coverage, and published localisation notes in `docs/dev/kisa_connector_notes.md` + fixture guidance for Docs/QA.| | ||||
| |FEEDCONN-KISA-02-007 RSS contract & localisation brief|BE-Conn-KISA|Research|**DONE (2025-10-11)** – Documented RSS URLs, confirmed UTF-8 payload (no additional cookies required), and drafted localisation plan (Hangul glossary + optional MT plugin). Remaining open item: capture SPA detail API contract for full-text translations.| | ||||
		Reference in New Issue
	
	Block a user