Rename Concelier Source modules to Connector

This commit is contained in:
master
2025-10-18 20:11:18 +03:00
parent 89ede53cc3
commit 052da7a7d0
789 changed files with 1489 additions and 1489 deletions

View File

@@ -0,0 +1,343 @@
[
{
"advisoryKey": "kev/cve-2021-43798",
"affectedPackages": [
{
"type": "vendor",
"identifier": "Grafana Labs::Grafana",
"platform": null,
"versionRanges": [
{
"fixedVersion": null,
"introducedVersion": null,
"lastAffectedVersion": null,
"primitives": {
"evr": null,
"hasVendorExtensions": true,
"nevra": null,
"semVer": null,
"vendorExtensions": {
"kev.vendorProject": "Grafana Labs",
"kev.product": "Grafana",
"kev.requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.",
"kev.knownRansomwareCampaignUse": "Unknown",
"kev.notes": "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798",
"kev.catalogVersion": "2025.10.09",
"kev.catalogReleased": "2025-10-09T16:52:28.6547000+00:00",
"kev.dateAdded": "2025-10-09",
"kev.dueDate": "2025-10-30",
"kev.cwe": "CWE-22"
}
},
"provenance": {
"source": "kev",
"kind": "kev-range",
"value": "Grafana Labs::Grafana",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"rangeExpression": null,
"rangeKind": "vendor"
}
],
"normalizedVersions": [
{
"scheme": "kev.catalog",
"type": "exact",
"min": null,
"minInclusive": null,
"max": null,
"maxInclusive": null,
"value": "2025.10.09",
"notes": "Grafana Labs::Grafana"
},
{
"scheme": "kev.date-added",
"type": "exact",
"min": null,
"minInclusive": null,
"max": null,
"maxInclusive": null,
"value": "2025-10-09",
"notes": "Grafana Labs::Grafana"
},
{
"scheme": "kev.due-date",
"type": "lte",
"min": null,
"minInclusive": null,
"max": "2025-10-30",
"maxInclusive": true,
"value": null,
"notes": "Grafana Labs::Grafana"
}
],
"statuses": [],
"provenance": [
{
"source": "kev",
"kind": "mapping",
"value": "2025.10.09",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
}
]
}
],
"aliases": [
"CVE-2021-43798"
],
"credits": [],
"cvssMetrics": [],
"exploitKnown": true,
"language": "en",
"modified": "2025-10-09T16:52:28.6547+00:00",
"provenance": [
{
"source": "kev",
"kind": "document",
"value": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
"decisionReason": null,
"recordedAt": "2025-10-10T00:00:00+00:00",
"fieldMask": []
},
{
"source": "kev",
"kind": "mapping",
"value": "2025.10.09",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
}
],
"published": "2025-10-09T00:00:00+00:00",
"references": [
{
"kind": "reference",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2021-43798",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "kev.notes",
"summary": null,
"url": "https://grafana.com/security/advisory"
},
{
"kind": "reference",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2021-43798",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "kev.notes",
"summary": null,
"url": "https://nvd.nist.gov/vuln/detail/CVE-2021-43798"
},
{
"kind": "advisory",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2021-43798",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "cisa-kev",
"summary": null,
"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2021-43798"
},
{
"kind": "reference",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2021-43798",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "cisa-kev-feed",
"summary": null,
"url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
}
],
"severity": null,
"summary": "Grafana contains a path traversal vulnerability that could allow access to local files.",
"title": "Grafana Path Traversal Vulnerability"
},
{
"advisoryKey": "kev/cve-2024-12345",
"affectedPackages": [
{
"type": "vendor",
"identifier": "Acme Corp::Acme Widget",
"platform": null,
"versionRanges": [
{
"fixedVersion": null,
"introducedVersion": null,
"lastAffectedVersion": null,
"primitives": {
"evr": null,
"hasVendorExtensions": true,
"nevra": null,
"semVer": null,
"vendorExtensions": {
"kev.vendorProject": "Acme Corp",
"kev.product": "Acme Widget",
"kev.requiredAction": "Apply vendor patch KB-1234.",
"kev.knownRansomwareCampaignUse": "Confirmed",
"kev.notes": "https://acme.example/advisories/KB-1234 https://nvd.nist.gov/vuln/detail/CVE-2024-12345 additional context ignored",
"kev.catalogVersion": "2025.10.09",
"kev.catalogReleased": "2025-10-09T16:52:28.6547000+00:00",
"kev.dateAdded": "2025-08-01",
"kev.cwe": "CWE-120,CWE-787"
}
},
"provenance": {
"source": "kev",
"kind": "kev-range",
"value": "Acme Corp::Acme Widget",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"rangeExpression": null,
"rangeKind": "vendor"
}
],
"normalizedVersions": [
{
"scheme": "kev.catalog",
"type": "exact",
"min": null,
"minInclusive": null,
"max": null,
"maxInclusive": null,
"value": "2025.10.09",
"notes": "Acme Corp::Acme Widget"
},
{
"scheme": "kev.date-added",
"type": "exact",
"min": null,
"minInclusive": null,
"max": null,
"maxInclusive": null,
"value": "2025-08-01",
"notes": "Acme Corp::Acme Widget"
}
],
"statuses": [],
"provenance": [
{
"source": "kev",
"kind": "mapping",
"value": "2025.10.09",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
}
]
}
],
"aliases": [
"CVE-2024-12345"
],
"credits": [],
"cvssMetrics": [],
"exploitKnown": true,
"language": "en",
"modified": "2025-10-09T16:52:28.6547+00:00",
"provenance": [
{
"source": "kev",
"kind": "document",
"value": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
"decisionReason": null,
"recordedAt": "2025-10-10T00:00:00+00:00",
"fieldMask": []
},
{
"source": "kev",
"kind": "mapping",
"value": "2025.10.09",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
}
],
"published": "2025-08-01T00:00:00+00:00",
"references": [
{
"kind": "reference",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2024-12345",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "kev.notes",
"summary": null,
"url": "https://acme.example/advisories/KB-1234"
},
{
"kind": "reference",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2024-12345",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "kev.notes",
"summary": null,
"url": "https://nvd.nist.gov/vuln/detail/CVE-2024-12345"
},
{
"kind": "advisory",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2024-12345",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "cisa-kev",
"summary": null,
"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2024-12345"
},
{
"kind": "reference",
"provenance": {
"source": "kev",
"kind": "reference",
"value": "CVE-2024-12345",
"decisionReason": null,
"recordedAt": "2025-10-10T00:01:00+00:00",
"fieldMask": []
},
"sourceTag": "cisa-kev-feed",
"summary": null,
"url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
}
],
"severity": null,
"summary": "Acme Widget contains a buffer overflow that may allow remote code execution.",
"title": "Acme Widget Buffer Overflow"
}
]

View File

@@ -0,0 +1,38 @@
{
"title": "CISA Catalog of Known Exploited Vulnerabilities",
"catalogVersion": "2025.10.09",
"dateReleased": "2025-10-09T16:52:28.6547Z",
"count": 2,
"vulnerabilities": [
{
"cveID": "CVE-2021-43798",
"vendorProject": "Grafana Labs",
"product": "Grafana",
"vulnerabilityName": "Grafana Path Traversal Vulnerability",
"dateAdded": "2025-10-09",
"shortDescription": "Grafana contains a path traversal vulnerability that could allow access to local files.",
"requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.",
"dueDate": "2025-10-30",
"knownRansomwareCampaignUse": "Unknown",
"notes": "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798",
"cwes": [
"CWE-22"
]
},
{
"cveID": "CVE-2024-12345",
"vendorProject": "Acme Corp",
"product": "Acme Widget",
"vulnerabilityName": "Acme Widget Buffer Overflow",
"dateAdded": "2025-08-01",
"shortDescription": "Acme Widget contains a buffer overflow that may allow remote code execution.",
"requiredAction": "Apply vendor patch KB-1234.",
"knownRansomwareCampaignUse": "Confirmed",
"notes": "https://acme.example/advisories/KB-1234 https://nvd.nist.gov/vuln/detail/CVE-2024-12345 additional context ignored",
"cwes": [
"CWE-120",
"CWE-787"
]
}
]
}

View File

@@ -0,0 +1,218 @@
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Connector.Kev;
using StellaOps.Concelier.Connector.Kev.Configuration;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Testing;
using Xunit;
namespace StellaOps.Concelier.Connector.Kev.Tests;
[Collection("mongo-fixture")]
public sealed class KevConnectorTests : IAsyncLifetime
{
private static readonly Uri FeedUri = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json");
private const string CatalogEtag = "\"kev-2025-10-09\"";
private readonly MongoIntegrationFixture _fixture;
private readonly FakeTimeProvider _timeProvider;
private readonly CannedHttpMessageHandler _handler;
public KevConnectorTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero));
_handler = new CannedHttpMessageHandler();
}
[Fact]
public async Task FetchParseMap_ProducesDeterministicSnapshot()
{
await using var provider = await BuildServiceProviderAsync();
SeedCatalogResponse();
var connector = provider.GetRequiredService<KevConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
_timeProvider.Advance(TimeSpan.FromMinutes(1));
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
Assert.NotEmpty(advisories);
var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray();
var snapshot = SnapshotSerializer.ToSnapshot(ordered);
WriteOrAssertSnapshot(snapshot, "kev-advisories.snapshot.json");
var documentStore = provider.GetRequiredService<IDocumentStore>();
var document = await documentStore.FindBySourceAndUriAsync(KevConnectorPlugin.SourceName, FeedUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
SeedNotModifiedResponse();
await connector.FetchAsync(provider, CancellationToken.None);
_handler.AssertNoPendingResponses();
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(KevConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.Equal("2025.10.09", state!.Cursor.TryGetValue("catalogVersion", out var versionValue) ? versionValue.AsString : null);
Assert.True(state.Cursor.TryGetValue("catalogReleased", out var releasedValue) && releasedValue.BsonType is BsonType.DateTime);
Assert.True(IsEmptyArray(state.Cursor, "pendingDocuments"));
Assert.True(IsEmptyArray(state.Cursor, "pendingMappings"));
}
private async Task<ServiceProvider> BuildServiceProviderAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton<TimeProvider>(_timeProvider);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddKevConnector(options =>
{
options.FeedUri = FeedUri;
options.RequestTimeout = TimeSpan.FromSeconds(10);
});
services.Configure<HttpClientFactoryOptions>(KevOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler);
});
var provider = services.BuildServiceProvider();
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
return provider;
}
private void SeedCatalogResponse()
{
var payload = ReadFixture("kev-catalog.json");
_handler.AddResponse(FeedUri, () =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
};
response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag);
response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero);
return response;
});
}
private void SeedNotModifiedResponse()
{
_handler.AddResponse(FeedUri, () =>
{
var response = new HttpResponseMessage(HttpStatusCode.NotModified);
response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag);
return response;
});
}
private static bool IsEmptyArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return false;
}
return array.Count == 0;
}
private static string ReadFixture(string filename)
{
var path = GetExistingFixturePath(filename);
return File.ReadAllText(path);
}
private static void WriteOrAssertSnapshot(string snapshot, string filename)
{
if (ShouldUpdateFixtures())
{
var target = GetWritableFixturePath(filename);
File.WriteAllText(target, snapshot);
return;
}
var expected = ReadFixture(filename);
var normalizedExpected = Normalize(expected);
var normalizedSnapshot = Normalize(snapshot);
if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal))
{
var actualPath = Path.Combine(Path.GetDirectoryName(GetWritableFixturePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json");
File.WriteAllText(actualPath, snapshot);
}
Assert.Equal(normalizedExpected, normalizedSnapshot);
}
private static bool ShouldUpdateFixtures()
{
var value = Environment.GetEnvironmentVariable("UPDATE_KEV_FIXTURES");
return string.Equals(value, "1", StringComparison.Ordinal) || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase);
}
private static string Normalize(string value)
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
private static string GetExistingFixturePath(string filename)
{
var baseDir = AppContext.BaseDirectory;
var primary = Path.Combine(baseDir, "Source", "Kev", "Fixtures", filename);
if (File.Exists(primary))
{
return primary;
}
var fallback = Path.Combine(baseDir, "Kev", "Fixtures", filename);
if (File.Exists(fallback))
{
return fallback;
}
throw new FileNotFoundException($"Unable to locate KEV fixture '{filename}'.");
}
private static string GetWritableFixturePath(string filename)
{
var baseDir = AppContext.BaseDirectory;
var primaryDir = Path.Combine(baseDir, "Source", "Kev", "Fixtures");
Directory.CreateDirectory(primaryDir);
return Path.Combine(primaryDir, filename);
}
public Task InitializeAsync() => Task.CompletedTask;
public async Task DisposeAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
}
}

View File

@@ -0,0 +1,93 @@
using System;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Kev;
using StellaOps.Concelier.Connector.Kev.Internal;
using Xunit;
namespace StellaOps.Concelier.Connector.Kev.Tests;
public sealed class KevMapperTests
{
[Fact]
public void Map_BuildsVendorRangePrimitivesWithDueDate()
{
var catalog = new KevCatalogDto
{
CatalogVersion = "2025.10.09",
DateReleased = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero),
Vulnerabilities = new[]
{
new KevVulnerabilityDto
{
CveId = "CVE-2021-43798",
VendorProject = "Grafana Labs",
Product = "Grafana",
VulnerabilityName = "Grafana Path Traversal Vulnerability",
DateAdded = "2025-10-09",
ShortDescription = "Grafana contains a path traversal vulnerability that could allow access to local files.",
RequiredAction = "Apply mitigations per vendor instructions or discontinue use.",
DueDate = "2025-10-30",
KnownRansomwareCampaignUse = "Unknown",
Notes = "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798",
Cwes = new[] { "CWE-22" }
}
}
};
var feedUri = new Uri("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json");
var fetchedAt = new DateTimeOffset(2025, 10, 9, 17, 0, 0, TimeSpan.Zero);
var validatedAt = fetchedAt.AddMinutes(1);
var advisories = KevMapper.Map(catalog, KevConnectorPlugin.SourceName, feedUri, fetchedAt, validatedAt);
var advisory = Assert.Single(advisories);
Assert.True(advisory.ExploitKnown);
Assert.Contains("cve-2021-43798", advisory.Aliases, StringComparer.OrdinalIgnoreCase);
var affected = Assert.Single(advisory.AffectedPackages);
Assert.Equal(AffectedPackageTypes.Vendor, affected.Type);
Assert.Equal("Grafana Labs::Grafana", affected.Identifier);
Assert.Collection(
affected.NormalizedVersions,
rule =>
{
Assert.Equal("kev.catalog", rule.Scheme);
Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type);
Assert.Equal("2025.10.09", rule.Value);
Assert.Equal("Grafana Labs::Grafana", rule.Notes);
},
rule =>
{
Assert.Equal("kev.date-added", rule.Scheme);
Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type);
Assert.Equal("2025-10-09", rule.Value);
},
rule =>
{
Assert.Equal("kev.due-date", rule.Scheme);
Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, rule.Type);
Assert.Equal("2025-10-30", rule.Max);
Assert.True(rule.MaxInclusive);
});
var range = Assert.Single(affected.VersionRanges);
Assert.Equal(AffectedPackageTypes.Vendor, range.RangeKind);
var primitives = range.Primitives;
Assert.NotNull(primitives);
Assert.True(primitives!.HasVendorExtensions);
var extensions = primitives!.VendorExtensions!;
Assert.Equal("Grafana Labs", extensions["kev.vendorProject"]);
Assert.Equal("Grafana", extensions["kev.product"]);
Assert.Equal("2025-10-30", extensions["kev.dueDate"]);
Assert.Equal("Unknown", extensions["kev.knownRansomwareCampaignUse"]);
Assert.Equal("CWE-22", extensions["kev.cwe"]);
var references = advisory.References.Select(reference => reference.Url).ToArray();
Assert.Contains("https://grafana.com/security/advisory", references);
Assert.Contains("https://nvd.nist.gov/vuln/detail/CVE-2021-43798", references);
Assert.Contains("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2021-43798", references);
}
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Connector.Kev/StellaOps.Concelier.Connector.Kev.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="Kev/Fixtures/**" CopyToOutputDirectory="Always" />
</ItemGroup>
</Project>