nuget reorganization

This commit is contained in:
master
2025-11-18 23:45:25 +02:00
parent 77cee6a209
commit d3ecd7f8e6
7712 changed files with 13963 additions and 10007504 deletions

View File

@@ -25,7 +25,7 @@ public static class VexLinksetUpdatedEventFactory
var observationRefs = (observations ?? Enumerable.Empty<VexObservation>())
.Where(obs => obs is not null)
.SelectMany(obs => obs.Statements.Select(statement => new VexLinksetObservationRef(
.SelectMany(obs => obs.Statements.Select(statement => new VexLinksetObservationRefCore(
observationId: obs.ObservationId,
providerId: obs.ProviderId,
status: statement.Status.ToString().ToLowerInvariant(),
@@ -70,11 +70,11 @@ public static class VexLinksetUpdatedEventFactory
return value.Trim();
}
private sealed class VexLinksetObservationRefComparer : IEqualityComparer<VexLinksetObservationRef>
private sealed class VexLinksetObservationRefComparer : IEqualityComparer<VexLinksetObservationRefCore>
{
public static readonly VexLinksetObservationRefComparer Instance = new();
public bool Equals(VexLinksetObservationRef? x, VexLinksetObservationRef? y)
public bool Equals(VexLinksetObservationRefCore? x, VexLinksetObservationRefCore? y)
{
if (ReferenceEquals(x, y))
{
@@ -92,7 +92,7 @@ public static class VexLinksetUpdatedEventFactory
&& Nullable.Equals(x.Confidence, y.Confidence);
}
public int GetHashCode(VexLinksetObservationRef obj)
public int GetHashCode(VexLinksetObservationRefCore obj)
{
var hash = new HashCode();
hash.Add(obj.ObservationId, StringComparer.Ordinal);
@@ -137,18 +137,12 @@ public static class VexLinksetUpdatedEventFactory
}
}
public sealed record VexLinksetObservationRef(
string ObservationId,
string ProviderId,
string Status,
double? Confidence);
public sealed record VexLinksetUpdatedEvent(
string EventType,
string Tenant,
string LinksetId,
string VulnerabilityId,
string ProductKey,
ImmutableArray<VexLinksetObservationRef> Observations,
ImmutableArray<VexLinksetObservationRefCore> Observations,
ImmutableArray<VexObservationDisagreement> Disagreements,
DateTimeOffset CreatedAtUtc);

View File

@@ -1,5 +1,6 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using System.Collections.Immutable;
using System.Collections.Generic;
using System.Text.Json.Nodes;
namespace StellaOps.Excititor.Core.Observations;
@@ -360,7 +361,8 @@ public sealed record VexObservationLinkset
IEnumerable<string>? cpes,
IEnumerable<VexObservationReference>? references,
IEnumerable<string>? reconciledFrom = null,
IEnumerable<VexObservationDisagreement>? disagreements = null)
IEnumerable<VexObservationDisagreement>? disagreements = null,
IEnumerable<VexLinksetObservationRefModel>? observationRefs = null)
{
Aliases = NormalizeSet(aliases, toLower: true);
Purls = NormalizeSet(purls, toLower: false);
@@ -368,6 +370,7 @@ public sealed record VexObservationLinkset
References = NormalizeReferences(references);
ReconciledFrom = NormalizeSet(reconciledFrom, toLower: false);
Disagreements = NormalizeDisagreements(disagreements);
Observations = NormalizeObservationRefs(observationRefs);
}
public ImmutableArray<string> Aliases { get; }
@@ -381,6 +384,8 @@ public sealed record VexObservationLinkset
public ImmutableArray<string> ReconciledFrom { get; }
public ImmutableArray<VexObservationDisagreement> Disagreements { get; }
public ImmutableArray<VexLinksetObservationRefModel> Observations { get; }
private static ImmutableArray<string> NormalizeSet(IEnumerable<string>? values, bool toLower)
{
@@ -499,8 +504,39 @@ public sealed record VexObservationLinkset
return set.Count == 0 ? ImmutableArray<VexObservationDisagreement>.Empty : set.ToImmutableArray();
}
private static ImmutableArray<VexLinksetObservationRefModel> NormalizeObservationRefs(
IEnumerable<VexLinksetObservationRefModel>? refs)
{
if (refs is null)
{
return ImmutableArray<VexLinksetObservationRefModel>.Empty;
}
var set = new SortedSet<VexLinksetObservationRefModel>(VexLinksetObservationRefComparer.Instance);
foreach (var item in refs)
{
if (item is null)
{
continue;
}
var obsId = VexObservation.TrimToNull(item.ObservationId);
var provider = VexObservation.TrimToNull(item.ProviderId);
var status = VexObservation.TrimToNull(item.Status);
if (obsId is null || provider is null || status is null)
{
continue;
}
var clamped = item.Confidence is null ? null : Math.Clamp(item.Confidence.Value, 0.0, 1.0);
set.Add(new VexLinksetObservationRefModel(obsId, provider, status, clamped));
}
return set.Count == 0 ? ImmutableArray<VexLinksetObservationRefModel>.Empty : set.ToImmutableArray();
}
}
public sealed record VexObservationReference
{
public VexObservationReference(string type, string url)
@@ -536,3 +572,52 @@ public sealed record VexObservationDisagreement
public double? Confidence { get; }
}
public sealed record VexLinksetObservationRefModel(
string ObservationId,
string ProviderId,
string Status,
double? Confidence);
internal sealed class VexLinksetObservationRefComparer : IComparer<VexLinksetObservationRefModel>
{
public static readonly VexLinksetObservationRefComparer Instance = new();
public int Compare(VexLinksetObservationRefModel? x, VexLinksetObservationRefModel? y)
{
if (ReferenceEquals(x, y))
{
return 0;
}
if (x is null)
{
return -1;
}
if (y is null)
{
return 1;
}
var providerCompare = StringComparer.OrdinalIgnoreCase.Compare(x.ProviderId, y.ProviderId);
if (providerCompare != 0)
{
return providerCompare;
}
var statusCompare = StringComparer.OrdinalIgnoreCase.Compare(x.Status, y.Status);
if (statusCompare != 0)
{
return statusCompare;
}
var obsCompare = StringComparer.Ordinal.Compare(x.ObservationId, y.ObservationId);
if (obsCompare != 0)
{
return obsCompare;
}
return Nullable.Compare(x.Confidence, y.Confidence);
}
}

View File

@@ -65,6 +65,10 @@ internal sealed class VexObservationCollectionsMigration : IVexMongoMigration
.Ascending(x => x.Tenant)
.Ascending(x => x.ProductKey);
var tenantProviderIndex = Builders<VexLinksetRecord>.IndexKeys
.Ascending(x => x.Tenant)
.Ascending("ProviderIds");
var tenantDisagreementProviderIndex = Builders<VexLinksetRecord>.IndexKeys
.Ascending(x => x.Tenant)
.Ascending("Disagreements.ProviderId")
@@ -74,6 +78,7 @@ internal sealed class VexObservationCollectionsMigration : IVexMongoMigration
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantLinksetIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantVulnIndex), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantProductIndex), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantProviderIndex), cancellationToken: cancellationToken),
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantDisagreementProviderIndex), cancellationToken: cancellationToken));
}
}

View File

@@ -0,0 +1,229 @@
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using MongoDB.Driver;
using MongoDB.Bson;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Observations;
namespace StellaOps.Excititor.Storage.Mongo;
internal sealed class MongoVexObservationLookup : IVexObservationLookup
{
private readonly IMongoCollection<VexObservationRecord> _collection;
public MongoVexObservationLookup(IMongoDatabase database)
{
ArgumentNullException.ThrowIfNull(database);
_collection = database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations);
}
public async ValueTask<IReadOnlyList<VexObservation>> ListByTenantAsync(
string tenant,
CancellationToken cancellationToken)
{
var normalizedTenant = NormalizeTenant(tenant);
var filter = Builders<VexObservationRecord>.Filter.Eq(record => record.Tenant, normalizedTenant);
var records = await _collection
.Find(filter)
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt).Descending(r => r.ObservationId))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return records.Select(Map).ToList();
}
public async ValueTask<IReadOnlyList<VexObservation>> FindByFiltersAsync(
string tenant,
IReadOnlyCollection<string> observationIds,
IReadOnlyCollection<string> vulnerabilityIds,
IReadOnlyCollection<string> productKeys,
IReadOnlyCollection<string> purls,
IReadOnlyCollection<string> cpes,
IReadOnlyCollection<string> providerIds,
IReadOnlyCollection<VexClaimStatus> statuses,
VexObservationCursor? cursor,
int limit,
CancellationToken cancellationToken)
{
var normalizedTenant = NormalizeTenant(tenant);
var filters = new List<FilterDefinition<VexObservationRecord>>(capacity: 8)
{
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant)
};
AddInFilter(filters, r => r.ObservationId, observationIds);
AddInFilter(filters, r => r.VulnerabilityId, vulnerabilityIds.Select(v => v.ToLowerInvariant()));
AddInFilter(filters, r => r.ProductKey, productKeys.Select(p => p.ToLowerInvariant()));
AddInFilter(filters, r => r.ProviderId, providerIds.Select(p => p.ToLowerInvariant()));
if (statuses.Count > 0)
{
var statusStrings = statuses.Select(status => status.ToString().ToLowerInvariant()).ToArray();
filters.Add(Builders<VexObservationRecord>.Filter.In(r => r.Status, statusStrings));
}
if (cursor is not null)
{
var cursorFilter = Builders<VexObservationRecord>.Filter.Or(
Builders<VexObservationRecord>.Filter.Lt(r => r.CreatedAt, cursor.CreatedAtUtc.UtcDateTime),
Builders<VexObservationRecord>.Filter.And(
Builders<VexObservationRecord>.Filter.Eq(r => r.CreatedAt, cursor.CreatedAtUtc.UtcDateTime),
Builders<VexObservationRecord>.Filter.Lt(r => r.ObservationId, cursor.ObservationId)));
filters.Add(cursorFilter);
}
var combinedFilter = filters.Count == 1
? filters[0]
: Builders<VexObservationRecord>.Filter.And(filters);
var records = await _collection
.Find(combinedFilter)
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt).Descending(r => r.ObservationId))
.Limit(limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return records.Select(Map).ToList();
}
private static void AddInFilter(
ICollection<FilterDefinition<VexObservationRecord>> filters,
System.Linq.Expressions.Expression<Func<VexObservationRecord, string>> field,
IEnumerable<string> values)
{
var normalized = values
.Where(value => !string.IsNullOrWhiteSpace(value))
.Select(value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (normalized.Length > 0)
{
filters.Add(Builders<VexObservationRecord>.Filter.In(field, normalized));
}
}
private static VexObservation Map(VexObservationRecord record)
{
var statements = record.Statements.Select(MapStatement).ToImmutableArray();
var linkset = MapLinkset(record.Linkset);
var upstreamSignature = record.Upstream?.Signature is null
? new VexObservationSignature(false, null, null, null)
: new VexObservationSignature(
record.Upstream.Signature.Present,
record.Upstream.Signature.Subject,
record.Upstream.Signature.Issuer,
record.Upstream.Signature.VerifiedAt);
var upstream = record.Upstream is null
? new VexObservationUpstream(
upstreamId: record.ObservationId,
documentVersion: null,
fetchedAt: record.CreatedAt,
receivedAt: record.CreatedAt,
contentHash: record.Document.Digest,
signature: upstreamSignature)
: new VexObservationUpstream(
record.Upstream.UpstreamId,
record.Upstream.DocumentVersion,
record.Upstream.FetchedAt,
record.Upstream.ReceivedAt,
record.Upstream.ContentHash,
upstreamSignature);
var documentSignature = record.Document.Signature is null
? null
: new VexObservationSignature(
record.Document.Signature.Present,
record.Document.Signature.Subject,
record.Document.Signature.Issuer,
record.Document.Signature.VerifiedAt);
var content = record.Content is null
? new VexObservationContent("unknown", null, new JsonObject())
: new VexObservationContent(
record.Content.Format ?? "unknown",
record.Content.SpecVersion,
JsonNode.Parse(record.Content.Raw.ToJson()) ?? new JsonObject(),
metadata: ImmutableDictionary<string, string>.Empty);
var observation = new VexObservation(
observationId: record.ObservationId,
tenant: record.Tenant,
providerId: record.ProviderId,
streamId: string.IsNullOrWhiteSpace(record.StreamId) ? record.ProviderId : record.StreamId,
upstream: upstream,
statements: statements,
content: content,
linkset: linkset,
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
supersedes: ImmutableArray<string>.Empty,
attributes: ImmutableDictionary<string, string>.Empty);
return observation;
}
private static VexObservationStatement MapStatement(VexObservationStatementRecord record)
{
var justification = string.IsNullOrWhiteSpace(record.Justification)
? (VexJustification?)null
: Enum.Parse<VexJustification>(record.Justification, ignoreCase: true);
return new VexObservationStatement(
record.VulnerabilityId,
record.ProductKey,
Enum.Parse<VexClaimStatus>(record.Status, ignoreCase: true),
record.LastObserved,
locator: record.Locator,
justification: justification,
introducedVersion: record.IntroducedVersion,
fixedVersion: record.FixedVersion,
detail: record.Detail,
signals: new VexSignalSnapshot(
severity: record.ScopeScore.HasValue ? new VexSeveritySignal("scope", record.ScopeScore, "n/a", null) : null,
Kev: record.Kev,
Epss: record.Epss),
confidence: null,
metadata: ImmutableDictionary<string, string>.Empty,
evidence: null,
anchors: VexObservationAnchors.Empty,
additionalMetadata: ImmutableDictionary<string, string>.Empty,
signature: null);
}
private static VexObservationDisagreement MapDisagreement(VexLinksetDisagreementRecord record)
=> new(record.ProviderId, record.Status, record.Justification, record.Confidence);
private static VexObservationLinkset MapLinkset(VexObservationLinksetRecord record)
{
var aliases = record?.Aliases?.Where(NotNullOrWhiteSpace).Select(a => a.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var purls = record?.Purls?.Where(NotNullOrWhiteSpace).Select(p => p.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var cpes = record?.Cpes?.Where(NotNullOrWhiteSpace).Select(c => c.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var references = record?.References?.Select(r => new VexObservationReference(r.Type, r.Url)).ToImmutableArray() ?? ImmutableArray<VexObservationReference>.Empty;
var reconciledFrom = record?.ReconciledFrom?.Where(NotNullOrWhiteSpace).Select(r => r.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
var disagreements = record?.Disagreements?.Select(MapDisagreement).ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
var observationRefs = record?.Observations?.Select(o => new VexLinksetObservationRef(
o.ObservationId,
o.ProviderId,
o.Status,
o.Confidence)).ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRef>.Empty;
return new VexObservationLinkset(aliases, purls, cpes, references, reconciledFrom, disagreements, observationRefs);
}
private static bool NotNullOrWhiteSpace(string? value) => !string.IsNullOrWhiteSpace(value);
private static string NormalizeTenant(string tenant)
{
if (string.IsNullOrWhiteSpace(tenant))
{
throw new ArgumentException("tenant is required", nameof(tenant));
}
return tenant.Trim().ToLowerInvariant();
}
}

View File

@@ -57,6 +57,7 @@ public static class VexMongoServiceCollectionExtensions
services.AddScoped<IVexCacheMaintenance, MongoVexCacheMaintenance>();
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
services.AddScoped<VexStatementBackfillService>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
services.AddSingleton<IVexMongoMigration, VexInitialIndexMigration>();
services.AddSingleton<IVexMongoMigration, VexConsensusSignalsMigration>();
services.AddSingleton<IVexMongoMigration, VexConsensusHoldMigration>();

View File

@@ -292,13 +292,21 @@ internal sealed class VexObservationRecord
public string ProviderId { get; set; } = default!;
public string StreamId { get; set; } = string.Empty;
public string Status { get; set; } = default!;
public VexObservationDocumentRecord Document { get; set; } = new();
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
public VexObservationUpstreamRecord Upstream { get; set; } = new();
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
public VexObservationContentRecord Content { get; set; } = new();
public List<VexObservationStatementRecord> Statements { get; set; } = new();
public VexObservationLinksetRecord Linkset { get; set; } = new();
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
}
[BsonIgnoreExtraElements]
@@ -308,6 +316,122 @@ internal sealed class VexObservationDocumentRecord
public string? SourceUri { get; set; }
= null;
public string? Format { get; set; }
= null;
public string? Revision { get; set; }
= null;
public VexObservationSignatureRecord? Signature { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationSignatureRecord
{
public bool Present { get; set; }
= false;
public string? Subject { get; set; }
= null;
public string? Issuer { get; set; }
= null;
public DateTimeOffset? VerifiedAt { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationUpstreamRecord
{
public string UpstreamId { get; set; } = default!;
public string? DocumentVersion { get; set; }
= null;
public DateTimeOffset FetchedAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset ReceivedAt { get; set; } = DateTimeOffset.UtcNow;
public string ContentHash { get; set; } = default!;
public VexObservationSignatureRecord Signature { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationContentRecord
{
public string Format { get; set; } = "unknown";
public string? SpecVersion { get; set; }
= null;
public BsonDocument Raw { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationStatementRecord
{
public string VulnerabilityId { get; set; } = default!;
public string ProductKey { get; set; } = default!;
public string Status { get; set; } = default!;
public DateTimeOffset? LastObserved { get; set; }
= null;
public string? Locator { get; set; }
= null;
public string? Justification { get; set; }
= null;
public string? IntroducedVersion { get; set; }
= null;
public string? FixedVersion { get; set; }
= null;
public string? Detail { get; set; }
= null;
public double? ScopeScore { get; set; }
= null;
public double? Epss { get; set; }
= null;
public double? Kev { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationLinksetRecord
{
public List<string> Aliases { get; set; } = new();
public List<string> Purls { get; set; } = new();
public List<string> Cpes { get; set; } = new();
public List<VexObservationReferenceRecord> References { get; set; } = new();
public List<string> ReconciledFrom { get; set; } = new();
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
public List<VexObservationLinksetObservationRecord> Observations { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationReferenceRecord
{
public string Type { get; set; } = default!;
public string Url { get; set; } = default!;
}
[BsonIgnoreExtraElements]
@@ -324,6 +448,10 @@ internal sealed class VexLinksetRecord
public string ProductKey { get; set; } = default!;
public List<string> ProviderIds { get; set; } = new();
public List<string> Statuses { get; set; } = new();
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
@@ -343,6 +471,19 @@ internal sealed class VexLinksetDisagreementRecord
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexObservationLinksetObservationRecord
{
public string ObservationId { get; set; } = default!;
public string ProviderId { get; set; } = default!;
public string Status { get; set; } = default!;
public double? Confidence { get; set; }
= null;
}
[BsonIgnoreExtraElements]
internal sealed class VexProviderRecord
{