UP
This commit is contained in:
@@ -1,3 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
@@ -5,6 +8,7 @@ using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
|
||||
@@ -12,12 +16,20 @@ public sealed class AdvisoryStore : IAdvisoryStore
|
||||
{
|
||||
private readonly IMongoCollection<AdvisoryDocument> _collection;
|
||||
private readonly ILogger<AdvisoryStore> _logger;
|
||||
private readonly IAliasStore _aliasStore;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public AdvisoryStore(IMongoDatabase database, ILogger<AdvisoryStore> logger)
|
||||
public AdvisoryStore(
|
||||
IMongoDatabase database,
|
||||
IAliasStore aliasStore,
|
||||
ILogger<AdvisoryStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_collection = (database ?? throw new ArgumentNullException(nameof(database)))
|
||||
.GetCollection<AdvisoryDocument>(MongoStorageDefaults.Collections.Advisory);
|
||||
_aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +37,19 @@ public sealed class AdvisoryStore : IAdvisoryStore
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(advisory);
|
||||
|
||||
var missing = ProvenanceInspector.FindMissingProvenance(advisory);
|
||||
var primarySource = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown";
|
||||
foreach (var item in missing)
|
||||
{
|
||||
var source = string.IsNullOrWhiteSpace(item.Source) ? primarySource : item.Source;
|
||||
_logger.LogWarning(
|
||||
"Missing provenance detected for {Component} in advisory {AdvisoryKey} (source {Source}).",
|
||||
item.Component,
|
||||
advisory.AdvisoryKey,
|
||||
source);
|
||||
ProvenanceDiagnostics.RecordMissing(source, item.Component, item.RecordedAt);
|
||||
}
|
||||
|
||||
var payload = CanonicalJsonSerializer.Serialize(advisory);
|
||||
var document = new AdvisoryDocument
|
||||
{
|
||||
@@ -37,6 +62,10 @@ public sealed class AdvisoryStore : IAdvisoryStore
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
await _collection.ReplaceOneAsync(x => x.AdvisoryKey == advisory.AdvisoryKey, document, options, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey);
|
||||
|
||||
var aliasEntries = BuildAliasEntries(advisory);
|
||||
var updatedAt = _timeProvider.GetUtcNow();
|
||||
await _aliasStore.ReplaceAsync(advisory.AdvisoryKey, aliasEntries, updatedAt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
|
||||
@@ -49,6 +78,23 @@ public sealed class AdvisoryStore : IAdvisoryStore
|
||||
return document is null ? null : Deserialize(document.Payload);
|
||||
}
|
||||
|
||||
private static IEnumerable<AliasEntry> BuildAliasEntries(Advisory advisory)
|
||||
{
|
||||
foreach (var alias in advisory.Aliases)
|
||||
{
|
||||
if (AliasSchemeRegistry.TryGetScheme(alias, out var scheme))
|
||||
{
|
||||
yield return new AliasEntry(scheme, alias);
|
||||
}
|
||||
else
|
||||
{
|
||||
yield return new AliasEntry(AliasStoreConstants.UnscopedScheme, alias);
|
||||
}
|
||||
}
|
||||
|
||||
yield return new AliasEntry(AliasStoreConstants.PrimaryScheme, advisory.AdvisoryKey);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await _collection.Find(FilterDefinition<AdvisoryDocument>.Empty)
|
||||
@@ -182,8 +228,13 @@ public sealed class AdvisoryStore : IAdvisoryStore
|
||||
var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument
|
||||
? DeserializeProvenance(provenanceValue.AsBsonDocument)
|
||||
: AdvisoryProvenance.Empty;
|
||||
RangePrimitives? primitives = null;
|
||||
if (document.TryGetValue("primitives", out var primitivesValue) && primitivesValue.IsBsonDocument)
|
||||
{
|
||||
primitives = DeserializePrimitives(primitivesValue.AsBsonDocument);
|
||||
}
|
||||
|
||||
return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance);
|
||||
return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance, primitives);
|
||||
}
|
||||
|
||||
private static AffectedPackageStatus DeserializeStatus(BsonDocument document)
|
||||
@@ -225,6 +276,104 @@ public sealed class AdvisoryStore : IAdvisoryStore
|
||||
return new AdvisoryProvenance(source, kind, value ?? string.Empty, recordedAt ?? DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static RangePrimitives? DeserializePrimitives(BsonDocument document)
|
||||
{
|
||||
SemVerPrimitive? semVer = null;
|
||||
NevraPrimitive? nevra = null;
|
||||
EvrPrimitive? evr = null;
|
||||
IReadOnlyDictionary<string, string>? vendor = null;
|
||||
|
||||
if (document.TryGetValue("semVer", out var semverValue) && semverValue.IsBsonDocument)
|
||||
{
|
||||
var semverDoc = semverValue.AsBsonDocument;
|
||||
semVer = new SemVerPrimitive(
|
||||
semverDoc.TryGetValue("introduced", out var semIntroduced) && semIntroduced.IsString ? semIntroduced.AsString : null,
|
||||
semverDoc.TryGetValue("introducedInclusive", out var semIntroducedInclusive) && semIntroducedInclusive.IsBoolean && semIntroducedInclusive.AsBoolean,
|
||||
semverDoc.TryGetValue("fixed", out var semFixed) && semFixed.IsString ? semFixed.AsString : null,
|
||||
semverDoc.TryGetValue("fixedInclusive", out var semFixedInclusive) && semFixedInclusive.IsBoolean && semFixedInclusive.AsBoolean,
|
||||
semverDoc.TryGetValue("lastAffected", out var semLast) && semLast.IsString ? semLast.AsString : null,
|
||||
semverDoc.TryGetValue("lastAffectedInclusive", out var semLastInclusive) && semLastInclusive.IsBoolean && semLastInclusive.AsBoolean,
|
||||
semverDoc.TryGetValue("constraintExpression", out var constraint) && constraint.IsString ? constraint.AsString : null);
|
||||
}
|
||||
|
||||
if (document.TryGetValue("nevra", out var nevraValue) && nevraValue.IsBsonDocument)
|
||||
{
|
||||
var nevraDoc = nevraValue.AsBsonDocument;
|
||||
nevra = new NevraPrimitive(
|
||||
DeserializeNevraComponent(nevraDoc, "introduced"),
|
||||
DeserializeNevraComponent(nevraDoc, "fixed"),
|
||||
DeserializeNevraComponent(nevraDoc, "lastAffected"));
|
||||
}
|
||||
|
||||
if (document.TryGetValue("evr", out var evrValue) && evrValue.IsBsonDocument)
|
||||
{
|
||||
var evrDoc = evrValue.AsBsonDocument;
|
||||
evr = new EvrPrimitive(
|
||||
DeserializeEvrComponent(evrDoc, "introduced"),
|
||||
DeserializeEvrComponent(evrDoc, "fixed"),
|
||||
DeserializeEvrComponent(evrDoc, "lastAffected"));
|
||||
}
|
||||
|
||||
if (document.TryGetValue("vendorExtensions", out var vendorValue) && vendorValue.IsBsonDocument)
|
||||
{
|
||||
vendor = vendorValue.AsBsonDocument.Elements
|
||||
.Where(static e => e.Value.IsString)
|
||||
.ToDictionary(static e => e.Name, static e => e.Value.AsString, StringComparer.Ordinal);
|
||||
if (vendor.Count == 0)
|
||||
{
|
||||
vendor = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (semVer is null && nevra is null && evr is null && vendor is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new RangePrimitives(semVer, nevra, evr, vendor);
|
||||
}
|
||||
|
||||
private static NevraComponent? DeserializeNevraComponent(BsonDocument parent, string field)
|
||||
{
|
||||
if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var component = value.AsBsonDocument;
|
||||
var name = component.TryGetValue("name", out var nameValue) && nameValue.IsString ? nameValue.AsString : null;
|
||||
var version = component.TryGetValue("version", out var versionValue) && versionValue.IsString ? versionValue.AsString : null;
|
||||
if (name is null || version is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0;
|
||||
var release = component.TryGetValue("release", out var releaseValue) && releaseValue.IsString ? releaseValue.AsString : string.Empty;
|
||||
var architecture = component.TryGetValue("architecture", out var archValue) && archValue.IsString ? archValue.AsString : null;
|
||||
|
||||
return new NevraComponent(name, epoch, version, release, architecture);
|
||||
}
|
||||
|
||||
private static EvrComponent? DeserializeEvrComponent(BsonDocument parent, string field)
|
||||
{
|
||||
if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var component = value.AsBsonDocument;
|
||||
var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0;
|
||||
var upstream = component.TryGetValue("upstreamVersion", out var upstreamValue) && upstreamValue.IsString ? upstreamValue.AsString : null;
|
||||
if (upstream is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var revision = component.TryGetValue("revision", out var revisionValue) && revisionValue.IsString ? revisionValue.AsString : null;
|
||||
return new EvrComponent(epoch, upstream, revision);
|
||||
}
|
||||
|
||||
private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field)
|
||||
=> document.TryGetValue(field, out var value) ? TryConvertDateTime(value) : null;
|
||||
|
||||
|
||||
38
src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs
Normal file
38
src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs
Normal file
@@ -0,0 +1,38 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class AliasDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; set; }
|
||||
|
||||
[BsonElement("advisoryKey")]
|
||||
public string AdvisoryKey { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("scheme")]
|
||||
public string Scheme { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("value")]
|
||||
public string Value { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("updatedAt")]
|
||||
public DateTime UpdatedAt { get; set; }
|
||||
}
|
||||
|
||||
internal static class AliasDocumentExtensions
|
||||
{
|
||||
public static AliasRecord ToRecord(this AliasDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var updatedAt = DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc);
|
||||
return new AliasRecord(
|
||||
document.AdvisoryKey,
|
||||
document.Scheme,
|
||||
document.Value,
|
||||
new DateTimeOffset(updatedAt));
|
||||
}
|
||||
}
|
||||
157
src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs
Normal file
157
src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs
Normal file
@@ -0,0 +1,157 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
|
||||
public sealed class AliasStore : IAliasStore
|
||||
{
|
||||
private readonly IMongoCollection<AliasDocument> _collection;
|
||||
private readonly ILogger<AliasStore> _logger;
|
||||
|
||||
public AliasStore(IMongoDatabase database, ILogger<AliasStore> logger)
|
||||
{
|
||||
_collection = (database ?? throw new ArgumentNullException(nameof(database)))
|
||||
.GetCollection<AliasDocument>(MongoStorageDefaults.Collections.Alias);
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<AliasUpsertResult> ReplaceAsync(
|
||||
string advisoryKey,
|
||||
IEnumerable<AliasEntry> aliases,
|
||||
DateTimeOffset updatedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
|
||||
|
||||
var aliasList = Normalize(aliases).ToArray();
|
||||
var deleteFilter = Builders<AliasDocument>.Filter.Eq(x => x.AdvisoryKey, advisoryKey);
|
||||
await _collection.DeleteManyAsync(deleteFilter, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (aliasList.Length > 0)
|
||||
{
|
||||
var documents = new List<AliasDocument>(aliasList.Length);
|
||||
var updatedAtUtc = updatedAt.ToUniversalTime().UtcDateTime;
|
||||
foreach (var alias in aliasList)
|
||||
{
|
||||
documents.Add(new AliasDocument
|
||||
{
|
||||
Id = ObjectId.GenerateNewId(),
|
||||
AdvisoryKey = advisoryKey,
|
||||
Scheme = alias.Scheme,
|
||||
Value = alias.Value,
|
||||
UpdatedAt = updatedAtUtc,
|
||||
});
|
||||
}
|
||||
|
||||
if (documents.Count > 0)
|
||||
{
|
||||
await _collection.InsertManyAsync(
|
||||
documents,
|
||||
new InsertManyOptions { IsOrdered = false },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (aliasList.Length == 0)
|
||||
{
|
||||
return new AliasUpsertResult(advisoryKey, Array.Empty<AliasCollision>());
|
||||
}
|
||||
|
||||
var collisions = new List<AliasCollision>();
|
||||
foreach (var alias in aliasList)
|
||||
{
|
||||
var filter = Builders<AliasDocument>.Filter.Eq(x => x.Scheme, alias.Scheme)
|
||||
& Builders<AliasDocument>.Filter.Eq(x => x.Value, alias.Value);
|
||||
|
||||
using var cursor = await _collection.FindAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var advisoryKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var document in cursor.Current)
|
||||
{
|
||||
advisoryKeys.Add(document.AdvisoryKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (advisoryKeys.Count <= 1)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys.ToArray());
|
||||
collisions.Add(collision);
|
||||
AliasStoreMetrics.RecordCollision(alias.Scheme, advisoryKeys.Count);
|
||||
_logger.LogWarning(
|
||||
"Alias collision detected for {Scheme}:{Value}; advisories: {Advisories}",
|
||||
alias.Scheme,
|
||||
alias.Value,
|
||||
string.Join(", ", advisoryKeys));
|
||||
}
|
||||
|
||||
return new AliasUpsertResult(advisoryKey, collisions);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AliasRecord>> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(scheme);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(value);
|
||||
|
||||
var normalizedScheme = NormalizeScheme(scheme);
|
||||
var normalizedValue = value.Trim();
|
||||
var filter = Builders<AliasDocument>.Filter.Eq(x => x.Scheme, normalizedScheme)
|
||||
& Builders<AliasDocument>.Filter.Eq(x => x.Value, normalizedValue);
|
||||
|
||||
var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
return documents.Select(static d => d.ToRecord()).ToArray();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AliasRecord>> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
|
||||
var filter = Builders<AliasDocument>.Filter.Eq(x => x.AdvisoryKey, advisoryKey);
|
||||
var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
return documents.Select(static d => d.ToRecord()).ToArray();
|
||||
}
|
||||
|
||||
private static IEnumerable<AliasEntry> Normalize(IEnumerable<AliasEntry> aliases)
|
||||
{
|
||||
if (aliases is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var alias in aliases)
|
||||
{
|
||||
if (alias is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var scheme = NormalizeScheme(alias.Scheme);
|
||||
var value = alias.Value?.Trim();
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = $"{scheme}\u0001{value}";
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new AliasEntry(scheme, value);
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeScheme(string scheme)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(scheme)
|
||||
? AliasStoreConstants.UnscopedScheme
|
||||
: scheme.Trim().ToUpperInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
|
||||
public static class AliasStoreConstants
|
||||
{
|
||||
public const string PrimaryScheme = "PRIMARY";
|
||||
public const string UnscopedScheme = "UNSCOPED";
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
|
||||
internal static class AliasStoreMetrics
|
||||
{
|
||||
private static readonly Meter Meter = new("StellaOps.Feedser.Merge");
|
||||
|
||||
internal static readonly Counter<long> AliasCollisionCounter = Meter.CreateCounter<long>(
|
||||
"feedser.merge.alias_conflict",
|
||||
unit: "count",
|
||||
description: "Number of alias collisions detected when the same alias maps to multiple advisories.");
|
||||
|
||||
public static void RecordCollision(string scheme, int advisoryCount)
|
||||
{
|
||||
AliasCollisionCounter.Add(
|
||||
1,
|
||||
new KeyValuePair<string, object?>("scheme", scheme),
|
||||
new KeyValuePair<string, object?>("advisory_count", advisoryCount));
|
||||
}
|
||||
}
|
||||
27
src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs
Normal file
27
src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
|
||||
public interface IAliasStore
|
||||
{
|
||||
Task<AliasUpsertResult> ReplaceAsync(
|
||||
string advisoryKey,
|
||||
IEnumerable<AliasEntry> aliases,
|
||||
DateTimeOffset updatedAt,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<AliasRecord>> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<AliasRecord>> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record AliasEntry(string Scheme, string Value);
|
||||
|
||||
public sealed record AliasRecord(string AdvisoryKey, string Scheme, string Value, DateTimeOffset UpdatedAt);
|
||||
|
||||
public sealed record AliasCollision(string Scheme, string Value, IReadOnlyList<string> AdvisoryKeys);
|
||||
|
||||
public sealed record AliasUpsertResult(string AdvisoryKey, IReadOnlyList<AliasCollision> Collisions);
|
||||
@@ -9,7 +9,7 @@ namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory;
|
||||
public sealed class ChangeHistoryDocument
|
||||
{
|
||||
[BsonId]
|
||||
public Guid Id { get; set; }
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("source")]
|
||||
public string SourceName { get; set; } = string.Empty;
|
||||
@@ -18,7 +18,7 @@ public sealed class ChangeHistoryDocument
|
||||
public string AdvisoryKey { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("documentId")]
|
||||
public Guid DocumentId { get; set; }
|
||||
public string DocumentId { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("documentSha256")]
|
||||
public string DocumentSha256 { get; set; } = string.Empty;
|
||||
|
||||
@@ -22,10 +22,10 @@ internal static class ChangeHistoryDocumentExtensions
|
||||
|
||||
return new ChangeHistoryDocument
|
||||
{
|
||||
Id = record.Id,
|
||||
Id = record.Id.ToString(),
|
||||
SourceName = record.SourceName,
|
||||
AdvisoryKey = record.AdvisoryKey,
|
||||
DocumentId = record.DocumentId,
|
||||
DocumentId = record.DocumentId.ToString(),
|
||||
DocumentSha256 = record.DocumentSha256,
|
||||
CurrentHash = record.CurrentHash,
|
||||
PreviousHash = record.PreviousHash,
|
||||
@@ -55,10 +55,10 @@ internal static class ChangeHistoryDocumentExtensions
|
||||
var capturedAtUtc = DateTime.SpecifyKind(document.CapturedAt, DateTimeKind.Utc);
|
||||
|
||||
return new ChangeHistoryRecord(
|
||||
document.Id,
|
||||
Guid.Parse(document.Id),
|
||||
document.SourceName,
|
||||
document.AdvisoryKey,
|
||||
document.DocumentId,
|
||||
Guid.Parse(document.DocumentId),
|
||||
document.DocumentSha256,
|
||||
document.CurrentHash,
|
||||
document.PreviousHash,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
@@ -7,7 +8,7 @@ namespace StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
public sealed class DocumentDocument
|
||||
{
|
||||
[BsonId]
|
||||
public Guid Id { get; set; }
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("sourceName")]
|
||||
public string SourceName { get; set; } = string.Empty;
|
||||
@@ -59,7 +60,7 @@ internal static class DocumentDocumentExtensions
|
||||
{
|
||||
return new DocumentDocument
|
||||
{
|
||||
Id = record.Id,
|
||||
Id = record.Id.ToString(),
|
||||
SourceName = record.SourceName,
|
||||
Uri = record.Uri,
|
||||
FetchedAt = record.FetchedAt.UtcDateTime,
|
||||
@@ -96,7 +97,7 @@ internal static class DocumentDocumentExtensions
|
||||
}
|
||||
|
||||
return new DocumentRecord(
|
||||
document.Id,
|
||||
Guid.Parse(document.Id),
|
||||
document.SourceName,
|
||||
document.Uri,
|
||||
DateTime.SpecifyKind(document.FetchedAt, DateTimeKind.Utc),
|
||||
|
||||
@@ -48,7 +48,8 @@ public sealed class DocumentStore : IDocumentStore
|
||||
|
||||
public async Task<DocumentRecord?> FindAsync(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = await _collection.Find(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
var idValue = id.ToString();
|
||||
var document = await _collection.Find(x => x.Id == idValue).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return document?.ToRecord();
|
||||
}
|
||||
|
||||
@@ -60,7 +61,8 @@ public sealed class DocumentStore : IDocumentStore
|
||||
.Set(x => x.Status, status)
|
||||
.Set(x => x.LastModified, DateTime.UtcNow);
|
||||
|
||||
var result = await _collection.UpdateOneAsync(x => x.Id == id, update, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var idValue = id.ToString();
|
||||
var result = await _collection.UpdateOneAsync(x => x.Id == idValue, update, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
return result.MatchedCount > 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
@@ -7,10 +8,10 @@ namespace StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
public sealed class DtoDocument
|
||||
{
|
||||
[BsonId]
|
||||
public Guid Id { get; set; }
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("documentId")]
|
||||
public Guid DocumentId { get; set; }
|
||||
public string DocumentId { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("sourceName")]
|
||||
public string SourceName { get; set; } = string.Empty;
|
||||
@@ -30,8 +31,8 @@ internal static class DtoDocumentExtensions
|
||||
public static DtoDocument FromRecord(DtoRecord record)
|
||||
=> new()
|
||||
{
|
||||
Id = record.Id,
|
||||
DocumentId = record.DocumentId,
|
||||
Id = record.Id.ToString(),
|
||||
DocumentId = record.DocumentId.ToString(),
|
||||
SourceName = record.SourceName,
|
||||
SchemaVersion = record.SchemaVersion,
|
||||
Payload = record.Payload ?? new BsonDocument(),
|
||||
@@ -40,8 +41,8 @@ internal static class DtoDocumentExtensions
|
||||
|
||||
public static DtoRecord ToRecord(this DtoDocument document)
|
||||
=> new(
|
||||
document.Id,
|
||||
document.DocumentId,
|
||||
Guid.Parse(document.Id),
|
||||
Guid.Parse(document.DocumentId),
|
||||
document.SourceName,
|
||||
document.SchemaVersion,
|
||||
document.Payload,
|
||||
|
||||
@@ -20,7 +20,8 @@ public sealed class DtoStore : IDtoStore
|
||||
ArgumentNullException.ThrowIfNull(record);
|
||||
|
||||
var document = DtoDocumentExtensions.FromRecord(record);
|
||||
var filter = Builders<DtoDocument>.Filter.Eq(x => x.DocumentId, record.DocumentId)
|
||||
var documentId = record.DocumentId.ToString();
|
||||
var filter = Builders<DtoDocument>.Filter.Eq(x => x.DocumentId, documentId)
|
||||
& Builders<DtoDocument>.Filter.Eq(x => x.SourceName, record.SourceName);
|
||||
|
||||
var options = new FindOneAndReplaceOptions<DtoDocument>
|
||||
@@ -36,7 +37,8 @@ public sealed class DtoStore : IDtoStore
|
||||
|
||||
public async Task<DtoRecord?> FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = await _collection.Find(x => x.DocumentId == documentId)
|
||||
var documentIdValue = documentId.ToString();
|
||||
var document = await _collection.Find(x => x.DocumentId == documentIdValue)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return document?.ToRecord();
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.Exporting;
|
||||
@@ -31,6 +33,21 @@ public sealed class ExportStateDocument
|
||||
|
||||
[BsonElement("updatedAt")]
|
||||
public DateTime UpdatedAt { get; set; }
|
||||
|
||||
[BsonElement("files")]
|
||||
public List<ExportStateFileDocument>? Files { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ExportStateFileDocument
|
||||
{
|
||||
[BsonElement("path")]
|
||||
public string Path { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("length")]
|
||||
public long Length { get; set; }
|
||||
|
||||
[BsonElement("digest")]
|
||||
public string Digest { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
internal static class ExportStateDocumentExtensions
|
||||
@@ -47,6 +64,12 @@ internal static class ExportStateDocumentExtensions
|
||||
TargetRepository = record.TargetRepository,
|
||||
ExporterVersion = record.ExporterVersion,
|
||||
UpdatedAt = record.UpdatedAt.UtcDateTime,
|
||||
Files = record.Files.Select(static file => new ExportStateFileDocument
|
||||
{
|
||||
Path = file.Path,
|
||||
Length = file.Length,
|
||||
Digest = file.Digest,
|
||||
}).ToList(),
|
||||
};
|
||||
|
||||
public static ExportStateRecord ToRecord(this ExportStateDocument document)
|
||||
@@ -59,5 +82,9 @@ internal static class ExportStateDocumentExtensions
|
||||
document.ExportCursor,
|
||||
document.TargetRepository,
|
||||
document.ExporterVersion,
|
||||
DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc));
|
||||
DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc),
|
||||
(document.Files ?? new List<ExportStateFileDocument>())
|
||||
.Where(static entry => !string.IsNullOrWhiteSpace(entry.Path))
|
||||
.Select(static entry => new ExportFileRecord(entry.Path, entry.Length, entry.Digest))
|
||||
.ToArray());
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
@@ -32,12 +33,14 @@ public sealed class ExportStateManager
|
||||
string? targetRepository,
|
||||
string exporterVersion,
|
||||
bool resetBaseline,
|
||||
IReadOnlyList<ExportFileRecord> manifest,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(exporterId);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exportId);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exportDigest);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
|
||||
manifest ??= Array.Empty<ExportFileRecord>();
|
||||
|
||||
var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
@@ -55,7 +58,8 @@ public sealed class ExportStateManager
|
||||
ExportCursor: cursor ?? exportDigest,
|
||||
TargetRepository: resolvedRepository,
|
||||
ExporterVersion: exporterVersion,
|
||||
UpdatedAt: now),
|
||||
UpdatedAt: now,
|
||||
Files: manifest),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -81,6 +85,7 @@ public sealed class ExportStateManager
|
||||
TargetRepository = resolvedRepo,
|
||||
ExporterVersion = exporterVersion,
|
||||
UpdatedAt = now,
|
||||
Files = manifest,
|
||||
}
|
||||
: existing with
|
||||
{
|
||||
@@ -90,6 +95,7 @@ public sealed class ExportStateManager
|
||||
TargetRepository = resolvedRepo,
|
||||
ExporterVersion = exporterVersion,
|
||||
UpdatedAt = now,
|
||||
Files = manifest,
|
||||
};
|
||||
|
||||
return await _store.UpsertAsync(updatedRecord, cancellationToken).ConfigureAwait(false);
|
||||
@@ -100,11 +106,13 @@ public sealed class ExportStateManager
|
||||
string deltaDigest,
|
||||
string? cursor,
|
||||
string exporterVersion,
|
||||
IReadOnlyList<ExportFileRecord> manifest,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(exporterId);
|
||||
ArgumentException.ThrowIfNullOrEmpty(deltaDigest);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
|
||||
manifest ??= Array.Empty<ExportFileRecord>();
|
||||
|
||||
var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is null)
|
||||
@@ -119,6 +127,7 @@ public sealed class ExportStateManager
|
||||
ExportCursor = cursor ?? existing.ExportCursor,
|
||||
ExporterVersion = exporterVersion,
|
||||
UpdatedAt = now,
|
||||
Files = manifest,
|
||||
};
|
||||
|
||||
return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -9,4 +9,7 @@ public sealed record ExportStateRecord(
|
||||
string? ExportCursor,
|
||||
string? TargetRepository,
|
||||
string? ExporterVersion,
|
||||
DateTimeOffset UpdatedAt);
|
||||
DateTimeOffset UpdatedAt,
|
||||
IReadOnlyList<ExportFileRecord> Files);
|
||||
|
||||
public sealed record ExportFileRecord(string Path, long Length, string Digest);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
@@ -11,8 +12,7 @@ namespace StellaOps.Feedser.Storage.Mongo;
|
||||
public sealed class JobRunDocument
|
||||
{
|
||||
[BsonId]
|
||||
[BsonGuidRepresentation(GuidRepresentation.Standard)]
|
||||
public Guid Id { get; set; }
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("kind")]
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
@@ -60,7 +60,7 @@ internal static class JobRunDocumentExtensions
|
||||
{
|
||||
return new JobRunDocument
|
||||
{
|
||||
Id = id,
|
||||
Id = id.ToString(),
|
||||
Kind = request.Kind,
|
||||
Status = JobRunStatus.Pending.ToString(),
|
||||
Trigger = request.Trigger,
|
||||
@@ -79,7 +79,7 @@ internal static class JobRunDocumentExtensions
|
||||
var parameters = document.Parameters?.ToDictionary() ?? new Dictionary<string, object?>();
|
||||
|
||||
return new JobRunSnapshot(
|
||||
document.Id,
|
||||
Guid.Parse(document.Id),
|
||||
document.Kind,
|
||||
Enum.Parse<JobRunStatus>(document.Status, ignoreCase: true),
|
||||
DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc),
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Feedser.Storage.Mongo.MergeEvents;
|
||||
@@ -6,7 +10,7 @@ namespace StellaOps.Feedser.Storage.Mongo.MergeEvents;
|
||||
public sealed class MergeEventDocument
|
||||
{
|
||||
[BsonId]
|
||||
public Guid Id { get; set; }
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("advisoryKey")]
|
||||
public string AdvisoryKey { get; set; } = string.Empty;
|
||||
@@ -21,7 +25,7 @@ public sealed class MergeEventDocument
|
||||
public DateTime MergedAt { get; set; }
|
||||
|
||||
[BsonElement("inputDocuments")]
|
||||
public List<Guid> InputDocuments { get; set; } = new();
|
||||
public List<string> InputDocuments { get; set; } = new();
|
||||
}
|
||||
|
||||
internal static class MergeEventDocumentExtensions
|
||||
@@ -29,20 +33,20 @@ internal static class MergeEventDocumentExtensions
|
||||
public static MergeEventDocument FromRecord(MergeEventRecord record)
|
||||
=> new()
|
||||
{
|
||||
Id = record.Id,
|
||||
Id = record.Id.ToString(),
|
||||
AdvisoryKey = record.AdvisoryKey,
|
||||
BeforeHash = record.BeforeHash,
|
||||
AfterHash = record.AfterHash,
|
||||
MergedAt = record.MergedAt.UtcDateTime,
|
||||
InputDocuments = record.InputDocumentIds.ToList(),
|
||||
InputDocuments = record.InputDocumentIds.Select(static id => id.ToString()).ToList(),
|
||||
};
|
||||
|
||||
public static MergeEventRecord ToRecord(this MergeEventDocument document)
|
||||
=> new(
|
||||
document.Id,
|
||||
Guid.Parse(document.Id),
|
||||
document.AdvisoryKey,
|
||||
document.BeforeHash,
|
||||
document.AfterHash,
|
||||
DateTime.SpecifyKind(document.MergedAt, DateTimeKind.Utc),
|
||||
document.InputDocuments);
|
||||
document.InputDocuments.Select(static value => Guid.Parse(value)).ToList());
|
||||
}
|
||||
|
||||
@@ -269,20 +269,22 @@ public sealed class MongoBootstrapper
|
||||
return collection.Indexes.CreateManyAsync(indexes, cancellationToken);
|
||||
}
|
||||
|
||||
private Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken)
|
||||
private async Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = _database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.PsirtFlags);
|
||||
var indexes = new List<CreateIndexModel<BsonDocument>>
|
||||
try
|
||||
{
|
||||
new(
|
||||
Builders<BsonDocument>.IndexKeys.Ascending("advisoryKey"),
|
||||
new CreateIndexOptions { Name = "psirt_advisoryKey_unique", Unique = true }),
|
||||
new(
|
||||
Builders<BsonDocument>.IndexKeys.Ascending("vendor"),
|
||||
new CreateIndexOptions { Name = "psirt_vendor" }),
|
||||
};
|
||||
await collection.Indexes.DropOneAsync("psirt_advisoryKey_unique", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (MongoCommandException ex) when (ex.CodeName == "IndexNotFound")
|
||||
{
|
||||
}
|
||||
|
||||
return collection.Indexes.CreateManyAsync(indexes, cancellationToken);
|
||||
var index = new CreateIndexModel<BsonDocument>(
|
||||
Builders<BsonDocument>.IndexKeys.Ascending("vendor"),
|
||||
new CreateIndexOptions { Name = "psirt_vendor" });
|
||||
|
||||
await collection.Indexes.CreateOneAsync(index, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private Task EnsureChangeHistoryIndexesAsync(CancellationToken cancellationToken)
|
||||
|
||||
@@ -36,7 +36,8 @@ public sealed class MongoJobStore : IJobStore
|
||||
|
||||
public async Task<JobRunSnapshot?> TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runId)
|
||||
var runIdValue = runId.ToString();
|
||||
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runIdValue)
|
||||
& Builders<JobRunDocument>.Filter.Eq(x => x.Status, PendingStatus);
|
||||
|
||||
var update = Builders<JobRunDocument>.Update
|
||||
@@ -63,7 +64,8 @@ public sealed class MongoJobStore : IJobStore
|
||||
|
||||
public async Task<JobRunSnapshot?> TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runId)
|
||||
var runIdValue = runId.ToString();
|
||||
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runIdValue)
|
||||
& Builders<JobRunDocument>.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus });
|
||||
|
||||
var update = Builders<JobRunDocument>.Update
|
||||
@@ -91,7 +93,7 @@ public sealed class MongoJobStore : IJobStore
|
||||
|
||||
public async Task<JobRunSnapshot?> FindAsync(Guid runId, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await _collection.FindAsync(x => x.Id == runId, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var cursor = await _collection.FindAsync(x => x.Id == runId.ToString(), cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return document?.ToSnapshot();
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Aliases;
|
||||
using StellaOps.Feedser.Storage.Mongo.ChangeHistory;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
@@ -58,6 +59,7 @@ public static class ServiceCollectionExtensions
|
||||
services.AddSingleton<IDocumentStore, DocumentStore>();
|
||||
services.AddSingleton<IDtoStore, DtoStore>();
|
||||
services.AddSingleton<IAdvisoryStore, AdvisoryStore>();
|
||||
services.AddSingleton<IAliasStore, AliasStore>();
|
||||
services.AddSingleton<IChangeHistoryStore, MongoChangeHistoryStore>();
|
||||
services.AddSingleton<IJpFlagStore, JpFlagStore>();
|
||||
services.AddSingleton<IPsirtFlagStore, PsirtFlagStore>();
|
||||
|
||||
@@ -13,3 +13,4 @@
|
||||
|Migration playbook for schema/index changes|BE-Storage|Storage.Mongo|DONE – `MongoMigrationRunner` executes `IMongoMigration` steps recorded in `schema_migrations`; see `MIGRATIONS.md`.|
|
||||
|Raw document retention/TTL strategy|BE-Storage|Storage.Mongo|DONE – retention options flow into `RawDocumentRetentionService` and TTL migrations for `document`/GridFS indexes.|
|
||||
|Persist last failure reason in SourceState|BE-Storage|Storage.Mongo|DONE – `MongoSourceStateRepository.MarkFailureAsync` stores `lastFailureReason` with length guard + reset on success.|
|
||||
|AdvisoryStore range primitives deserialization|BE-Storage|Models|DONE – BSON helpers handle `RangePrimitives`; regression test covers SemVer/NEVRA/EVR envelopes persisted through Mongo.|
|
||||
|
||||
Reference in New Issue
Block a user