UP
Some checks failed
Build Test Deploy / build-test (push) Has been cancelled
Build Test Deploy / docs (push) Has been cancelled
Build Test Deploy / deploy (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
Vladimir Moushkov
2025-10-09 18:59:17 +03:00
parent 18b1922f60
commit d0c95cf328
277 changed files with 17449 additions and 595 deletions

View File

@@ -1,3 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Text.Json.Serialization;
@@ -5,6 +8,7 @@ using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Storage.Mongo.Aliases;
namespace StellaOps.Feedser.Storage.Mongo.Advisories;
@@ -12,12 +16,20 @@ public sealed class AdvisoryStore : IAdvisoryStore
{
private readonly IMongoCollection<AdvisoryDocument> _collection;
private readonly ILogger<AdvisoryStore> _logger;
private readonly IAliasStore _aliasStore;
private readonly TimeProvider _timeProvider;
public AdvisoryStore(IMongoDatabase database, ILogger<AdvisoryStore> logger)
public AdvisoryStore(
IMongoDatabase database,
IAliasStore aliasStore,
ILogger<AdvisoryStore> logger,
TimeProvider? timeProvider = null)
{
_collection = (database ?? throw new ArgumentNullException(nameof(database)))
.GetCollection<AdvisoryDocument>(MongoStorageDefaults.Collections.Advisory);
_aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
@@ -25,6 +37,19 @@ public sealed class AdvisoryStore : IAdvisoryStore
{
ArgumentNullException.ThrowIfNull(advisory);
var missing = ProvenanceInspector.FindMissingProvenance(advisory);
var primarySource = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown";
foreach (var item in missing)
{
var source = string.IsNullOrWhiteSpace(item.Source) ? primarySource : item.Source;
_logger.LogWarning(
"Missing provenance detected for {Component} in advisory {AdvisoryKey} (source {Source}).",
item.Component,
advisory.AdvisoryKey,
source);
ProvenanceDiagnostics.RecordMissing(source, item.Component, item.RecordedAt);
}
var payload = CanonicalJsonSerializer.Serialize(advisory);
var document = new AdvisoryDocument
{
@@ -37,6 +62,10 @@ public sealed class AdvisoryStore : IAdvisoryStore
var options = new ReplaceOptions { IsUpsert = true };
await _collection.ReplaceOneAsync(x => x.AdvisoryKey == advisory.AdvisoryKey, document, options, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey);
var aliasEntries = BuildAliasEntries(advisory);
var updatedAt = _timeProvider.GetUtcNow();
await _aliasStore.ReplaceAsync(advisory.AdvisoryKey, aliasEntries, updatedAt, cancellationToken).ConfigureAwait(false);
}
public async Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
@@ -49,6 +78,23 @@ public sealed class AdvisoryStore : IAdvisoryStore
return document is null ? null : Deserialize(document.Payload);
}
private static IEnumerable<AliasEntry> BuildAliasEntries(Advisory advisory)
{
foreach (var alias in advisory.Aliases)
{
if (AliasSchemeRegistry.TryGetScheme(alias, out var scheme))
{
yield return new AliasEntry(scheme, alias);
}
else
{
yield return new AliasEntry(AliasStoreConstants.UnscopedScheme, alias);
}
}
yield return new AliasEntry(AliasStoreConstants.PrimaryScheme, advisory.AdvisoryKey);
}
public async Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
{
var cursor = await _collection.Find(FilterDefinition<AdvisoryDocument>.Empty)
@@ -182,8 +228,13 @@ public sealed class AdvisoryStore : IAdvisoryStore
var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument
? DeserializeProvenance(provenanceValue.AsBsonDocument)
: AdvisoryProvenance.Empty;
RangePrimitives? primitives = null;
if (document.TryGetValue("primitives", out var primitivesValue) && primitivesValue.IsBsonDocument)
{
primitives = DeserializePrimitives(primitivesValue.AsBsonDocument);
}
return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance);
return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance, primitives);
}
private static AffectedPackageStatus DeserializeStatus(BsonDocument document)
@@ -225,6 +276,104 @@ public sealed class AdvisoryStore : IAdvisoryStore
return new AdvisoryProvenance(source, kind, value ?? string.Empty, recordedAt ?? DateTimeOffset.UtcNow);
}
private static RangePrimitives? DeserializePrimitives(BsonDocument document)
{
SemVerPrimitive? semVer = null;
NevraPrimitive? nevra = null;
EvrPrimitive? evr = null;
IReadOnlyDictionary<string, string>? vendor = null;
if (document.TryGetValue("semVer", out var semverValue) && semverValue.IsBsonDocument)
{
var semverDoc = semverValue.AsBsonDocument;
semVer = new SemVerPrimitive(
semverDoc.TryGetValue("introduced", out var semIntroduced) && semIntroduced.IsString ? semIntroduced.AsString : null,
semverDoc.TryGetValue("introducedInclusive", out var semIntroducedInclusive) && semIntroducedInclusive.IsBoolean && semIntroducedInclusive.AsBoolean,
semverDoc.TryGetValue("fixed", out var semFixed) && semFixed.IsString ? semFixed.AsString : null,
semverDoc.TryGetValue("fixedInclusive", out var semFixedInclusive) && semFixedInclusive.IsBoolean && semFixedInclusive.AsBoolean,
semverDoc.TryGetValue("lastAffected", out var semLast) && semLast.IsString ? semLast.AsString : null,
semverDoc.TryGetValue("lastAffectedInclusive", out var semLastInclusive) && semLastInclusive.IsBoolean && semLastInclusive.AsBoolean,
semverDoc.TryGetValue("constraintExpression", out var constraint) && constraint.IsString ? constraint.AsString : null);
}
if (document.TryGetValue("nevra", out var nevraValue) && nevraValue.IsBsonDocument)
{
var nevraDoc = nevraValue.AsBsonDocument;
nevra = new NevraPrimitive(
DeserializeNevraComponent(nevraDoc, "introduced"),
DeserializeNevraComponent(nevraDoc, "fixed"),
DeserializeNevraComponent(nevraDoc, "lastAffected"));
}
if (document.TryGetValue("evr", out var evrValue) && evrValue.IsBsonDocument)
{
var evrDoc = evrValue.AsBsonDocument;
evr = new EvrPrimitive(
DeserializeEvrComponent(evrDoc, "introduced"),
DeserializeEvrComponent(evrDoc, "fixed"),
DeserializeEvrComponent(evrDoc, "lastAffected"));
}
if (document.TryGetValue("vendorExtensions", out var vendorValue) && vendorValue.IsBsonDocument)
{
vendor = vendorValue.AsBsonDocument.Elements
.Where(static e => e.Value.IsString)
.ToDictionary(static e => e.Name, static e => e.Value.AsString, StringComparer.Ordinal);
if (vendor.Count == 0)
{
vendor = null;
}
}
if (semVer is null && nevra is null && evr is null && vendor is null)
{
return null;
}
return new RangePrimitives(semVer, nevra, evr, vendor);
}
private static NevraComponent? DeserializeNevraComponent(BsonDocument parent, string field)
{
if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument)
{
return null;
}
var component = value.AsBsonDocument;
var name = component.TryGetValue("name", out var nameValue) && nameValue.IsString ? nameValue.AsString : null;
var version = component.TryGetValue("version", out var versionValue) && versionValue.IsString ? versionValue.AsString : null;
if (name is null || version is null)
{
return null;
}
var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0;
var release = component.TryGetValue("release", out var releaseValue) && releaseValue.IsString ? releaseValue.AsString : string.Empty;
var architecture = component.TryGetValue("architecture", out var archValue) && archValue.IsString ? archValue.AsString : null;
return new NevraComponent(name, epoch, version, release, architecture);
}
private static EvrComponent? DeserializeEvrComponent(BsonDocument parent, string field)
{
if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument)
{
return null;
}
var component = value.AsBsonDocument;
var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0;
var upstream = component.TryGetValue("upstreamVersion", out var upstreamValue) && upstreamValue.IsString ? upstreamValue.AsString : null;
if (upstream is null)
{
return null;
}
var revision = component.TryGetValue("revision", out var revisionValue) && revisionValue.IsString ? revisionValue.AsString : null;
return new EvrComponent(epoch, upstream, revision);
}
private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field)
=> document.TryGetValue(field, out var value) ? TryConvertDateTime(value) : null;

View File

@@ -0,0 +1,38 @@
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
[BsonIgnoreExtraElements]
internal sealed class AliasDocument
{
[BsonId]
public ObjectId Id { get; set; }
[BsonElement("advisoryKey")]
public string AdvisoryKey { get; set; } = string.Empty;
[BsonElement("scheme")]
public string Scheme { get; set; } = string.Empty;
[BsonElement("value")]
public string Value { get; set; } = string.Empty;
[BsonElement("updatedAt")]
public DateTime UpdatedAt { get; set; }
}
internal static class AliasDocumentExtensions
{
public static AliasRecord ToRecord(this AliasDocument document)
{
ArgumentNullException.ThrowIfNull(document);
var updatedAt = DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc);
return new AliasRecord(
document.AdvisoryKey,
document.Scheme,
document.Value,
new DateTimeOffset(updatedAt));
}
}

View File

@@ -0,0 +1,157 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
public sealed class AliasStore : IAliasStore
{
private readonly IMongoCollection<AliasDocument> _collection;
private readonly ILogger<AliasStore> _logger;
public AliasStore(IMongoDatabase database, ILogger<AliasStore> logger)
{
_collection = (database ?? throw new ArgumentNullException(nameof(database)))
.GetCollection<AliasDocument>(MongoStorageDefaults.Collections.Alias);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<AliasUpsertResult> ReplaceAsync(
string advisoryKey,
IEnumerable<AliasEntry> aliases,
DateTimeOffset updatedAt,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
var aliasList = Normalize(aliases).ToArray();
var deleteFilter = Builders<AliasDocument>.Filter.Eq(x => x.AdvisoryKey, advisoryKey);
await _collection.DeleteManyAsync(deleteFilter, cancellationToken).ConfigureAwait(false);
if (aliasList.Length > 0)
{
var documents = new List<AliasDocument>(aliasList.Length);
var updatedAtUtc = updatedAt.ToUniversalTime().UtcDateTime;
foreach (var alias in aliasList)
{
documents.Add(new AliasDocument
{
Id = ObjectId.GenerateNewId(),
AdvisoryKey = advisoryKey,
Scheme = alias.Scheme,
Value = alias.Value,
UpdatedAt = updatedAtUtc,
});
}
if (documents.Count > 0)
{
await _collection.InsertManyAsync(
documents,
new InsertManyOptions { IsOrdered = false },
cancellationToken).ConfigureAwait(false);
}
}
if (aliasList.Length == 0)
{
return new AliasUpsertResult(advisoryKey, Array.Empty<AliasCollision>());
}
var collisions = new List<AliasCollision>();
foreach (var alias in aliasList)
{
var filter = Builders<AliasDocument>.Filter.Eq(x => x.Scheme, alias.Scheme)
& Builders<AliasDocument>.Filter.Eq(x => x.Value, alias.Value);
using var cursor = await _collection.FindAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false);
var advisoryKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
{
foreach (var document in cursor.Current)
{
advisoryKeys.Add(document.AdvisoryKey);
}
}
if (advisoryKeys.Count <= 1)
{
continue;
}
var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys.ToArray());
collisions.Add(collision);
AliasStoreMetrics.RecordCollision(alias.Scheme, advisoryKeys.Count);
_logger.LogWarning(
"Alias collision detected for {Scheme}:{Value}; advisories: {Advisories}",
alias.Scheme,
alias.Value,
string.Join(", ", advisoryKeys));
}
return new AliasUpsertResult(advisoryKey, collisions);
}
public async Task<IReadOnlyList<AliasRecord>> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scheme);
ArgumentException.ThrowIfNullOrWhiteSpace(value);
var normalizedScheme = NormalizeScheme(scheme);
var normalizedValue = value.Trim();
var filter = Builders<AliasDocument>.Filter.Eq(x => x.Scheme, normalizedScheme)
& Builders<AliasDocument>.Filter.Eq(x => x.Value, normalizedValue);
var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false);
return documents.Select(static d => d.ToRecord()).ToArray();
}
public async Task<IReadOnlyList<AliasRecord>> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
var filter = Builders<AliasDocument>.Filter.Eq(x => x.AdvisoryKey, advisoryKey);
var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false);
return documents.Select(static d => d.ToRecord()).ToArray();
}
private static IEnumerable<AliasEntry> Normalize(IEnumerable<AliasEntry> aliases)
{
if (aliases is null)
{
yield break;
}
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var alias in aliases)
{
if (alias is null)
{
continue;
}
var scheme = NormalizeScheme(alias.Scheme);
var value = alias.Value?.Trim();
if (string.IsNullOrEmpty(value))
{
continue;
}
var key = $"{scheme}\u0001{value}";
if (!seen.Add(key))
{
continue;
}
yield return new AliasEntry(scheme, value);
}
}
private static string NormalizeScheme(string scheme)
{
return string.IsNullOrWhiteSpace(scheme)
? AliasStoreConstants.UnscopedScheme
: scheme.Trim().ToUpperInvariant();
}
}

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
public static class AliasStoreConstants
{
public const string PrimaryScheme = "PRIMARY";
public const string UnscopedScheme = "UNSCOPED";
}

View File

@@ -0,0 +1,22 @@
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
internal static class AliasStoreMetrics
{
private static readonly Meter Meter = new("StellaOps.Feedser.Merge");
internal static readonly Counter<long> AliasCollisionCounter = Meter.CreateCounter<long>(
"feedser.merge.alias_conflict",
unit: "count",
description: "Number of alias collisions detected when the same alias maps to multiple advisories.");
public static void RecordCollision(string scheme, int advisoryCount)
{
AliasCollisionCounter.Add(
1,
new KeyValuePair<string, object?>("scheme", scheme),
new KeyValuePair<string, object?>("advisory_count", advisoryCount));
}
}

View File

@@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Feedser.Storage.Mongo.Aliases;
public interface IAliasStore
{
Task<AliasUpsertResult> ReplaceAsync(
string advisoryKey,
IEnumerable<AliasEntry> aliases,
DateTimeOffset updatedAt,
CancellationToken cancellationToken);
Task<IReadOnlyList<AliasRecord>> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken);
Task<IReadOnlyList<AliasRecord>> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken);
}
public sealed record AliasEntry(string Scheme, string Value);
public sealed record AliasRecord(string AdvisoryKey, string Scheme, string Value, DateTimeOffset UpdatedAt);
public sealed record AliasCollision(string Scheme, string Value, IReadOnlyList<string> AdvisoryKeys);
public sealed record AliasUpsertResult(string AdvisoryKey, IReadOnlyList<AliasCollision> Collisions);

View File

@@ -9,7 +9,7 @@ namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory;
public sealed class ChangeHistoryDocument
{
[BsonId]
public Guid Id { get; set; }
public string Id { get; set; } = string.Empty;
[BsonElement("source")]
public string SourceName { get; set; } = string.Empty;
@@ -18,7 +18,7 @@ public sealed class ChangeHistoryDocument
public string AdvisoryKey { get; set; } = string.Empty;
[BsonElement("documentId")]
public Guid DocumentId { get; set; }
public string DocumentId { get; set; } = string.Empty;
[BsonElement("documentSha256")]
public string DocumentSha256 { get; set; } = string.Empty;

View File

@@ -22,10 +22,10 @@ internal static class ChangeHistoryDocumentExtensions
return new ChangeHistoryDocument
{
Id = record.Id,
Id = record.Id.ToString(),
SourceName = record.SourceName,
AdvisoryKey = record.AdvisoryKey,
DocumentId = record.DocumentId,
DocumentId = record.DocumentId.ToString(),
DocumentSha256 = record.DocumentSha256,
CurrentHash = record.CurrentHash,
PreviousHash = record.PreviousHash,
@@ -55,10 +55,10 @@ internal static class ChangeHistoryDocumentExtensions
var capturedAtUtc = DateTime.SpecifyKind(document.CapturedAt, DateTimeKind.Utc);
return new ChangeHistoryRecord(
document.Id,
Guid.Parse(document.Id),
document.SourceName,
document.AdvisoryKey,
document.DocumentId,
Guid.Parse(document.DocumentId),
document.DocumentSha256,
document.CurrentHash,
document.PreviousHash,

View File

@@ -1,3 +1,4 @@
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
@@ -7,7 +8,7 @@ namespace StellaOps.Feedser.Storage.Mongo.Documents;
public sealed class DocumentDocument
{
[BsonId]
public Guid Id { get; set; }
public string Id { get; set; } = string.Empty;
[BsonElement("sourceName")]
public string SourceName { get; set; } = string.Empty;
@@ -59,7 +60,7 @@ internal static class DocumentDocumentExtensions
{
return new DocumentDocument
{
Id = record.Id,
Id = record.Id.ToString(),
SourceName = record.SourceName,
Uri = record.Uri,
FetchedAt = record.FetchedAt.UtcDateTime,
@@ -96,7 +97,7 @@ internal static class DocumentDocumentExtensions
}
return new DocumentRecord(
document.Id,
Guid.Parse(document.Id),
document.SourceName,
document.Uri,
DateTime.SpecifyKind(document.FetchedAt, DateTimeKind.Utc),

View File

@@ -48,7 +48,8 @@ public sealed class DocumentStore : IDocumentStore
public async Task<DocumentRecord?> FindAsync(Guid id, CancellationToken cancellationToken)
{
var document = await _collection.Find(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
var idValue = id.ToString();
var document = await _collection.Find(x => x.Id == idValue).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
return document?.ToRecord();
}
@@ -60,7 +61,8 @@ public sealed class DocumentStore : IDocumentStore
.Set(x => x.Status, status)
.Set(x => x.LastModified, DateTime.UtcNow);
var result = await _collection.UpdateOneAsync(x => x.Id == id, update, cancellationToken: cancellationToken).ConfigureAwait(false);
var idValue = id.ToString();
var result = await _collection.UpdateOneAsync(x => x.Id == idValue, update, cancellationToken: cancellationToken).ConfigureAwait(false);
return result.MatchedCount > 0;
}
}

View File

@@ -1,3 +1,4 @@
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
@@ -7,10 +8,10 @@ namespace StellaOps.Feedser.Storage.Mongo.Dtos;
public sealed class DtoDocument
{
[BsonId]
public Guid Id { get; set; }
public string Id { get; set; } = string.Empty;
[BsonElement("documentId")]
public Guid DocumentId { get; set; }
public string DocumentId { get; set; } = string.Empty;
[BsonElement("sourceName")]
public string SourceName { get; set; } = string.Empty;
@@ -30,8 +31,8 @@ internal static class DtoDocumentExtensions
public static DtoDocument FromRecord(DtoRecord record)
=> new()
{
Id = record.Id,
DocumentId = record.DocumentId,
Id = record.Id.ToString(),
DocumentId = record.DocumentId.ToString(),
SourceName = record.SourceName,
SchemaVersion = record.SchemaVersion,
Payload = record.Payload ?? new BsonDocument(),
@@ -40,8 +41,8 @@ internal static class DtoDocumentExtensions
public static DtoRecord ToRecord(this DtoDocument document)
=> new(
document.Id,
document.DocumentId,
Guid.Parse(document.Id),
Guid.Parse(document.DocumentId),
document.SourceName,
document.SchemaVersion,
document.Payload,

View File

@@ -20,7 +20,8 @@ public sealed class DtoStore : IDtoStore
ArgumentNullException.ThrowIfNull(record);
var document = DtoDocumentExtensions.FromRecord(record);
var filter = Builders<DtoDocument>.Filter.Eq(x => x.DocumentId, record.DocumentId)
var documentId = record.DocumentId.ToString();
var filter = Builders<DtoDocument>.Filter.Eq(x => x.DocumentId, documentId)
& Builders<DtoDocument>.Filter.Eq(x => x.SourceName, record.SourceName);
var options = new FindOneAndReplaceOptions<DtoDocument>
@@ -36,7 +37,8 @@ public sealed class DtoStore : IDtoStore
public async Task<DtoRecord?> FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken)
{
var document = await _collection.Find(x => x.DocumentId == documentId)
var documentIdValue = documentId.ToString();
var document = await _collection.Find(x => x.DocumentId == documentIdValue)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return document?.ToRecord();

View File

@@ -1,3 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Feedser.Storage.Mongo.Exporting;
@@ -31,6 +33,21 @@ public sealed class ExportStateDocument
[BsonElement("updatedAt")]
public DateTime UpdatedAt { get; set; }
[BsonElement("files")]
public List<ExportStateFileDocument>? Files { get; set; }
}
public sealed class ExportStateFileDocument
{
[BsonElement("path")]
public string Path { get; set; } = string.Empty;
[BsonElement("length")]
public long Length { get; set; }
[BsonElement("digest")]
public string Digest { get; set; } = string.Empty;
}
internal static class ExportStateDocumentExtensions
@@ -47,6 +64,12 @@ internal static class ExportStateDocumentExtensions
TargetRepository = record.TargetRepository,
ExporterVersion = record.ExporterVersion,
UpdatedAt = record.UpdatedAt.UtcDateTime,
Files = record.Files.Select(static file => new ExportStateFileDocument
{
Path = file.Path,
Length = file.Length,
Digest = file.Digest,
}).ToList(),
};
public static ExportStateRecord ToRecord(this ExportStateDocument document)
@@ -59,5 +82,9 @@ internal static class ExportStateDocumentExtensions
document.ExportCursor,
document.TargetRepository,
document.ExporterVersion,
DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc));
DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc),
(document.Files ?? new List<ExportStateFileDocument>())
.Where(static entry => !string.IsNullOrWhiteSpace(entry.Path))
.Select(static entry => new ExportFileRecord(entry.Path, entry.Length, entry.Digest))
.ToArray());
}

View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
@@ -32,12 +33,14 @@ public sealed class ExportStateManager
string? targetRepository,
string exporterVersion,
bool resetBaseline,
IReadOnlyList<ExportFileRecord> manifest,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrEmpty(exporterId);
ArgumentException.ThrowIfNullOrEmpty(exportId);
ArgumentException.ThrowIfNullOrEmpty(exportDigest);
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
manifest ??= Array.Empty<ExportFileRecord>();
var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
@@ -55,7 +58,8 @@ public sealed class ExportStateManager
ExportCursor: cursor ?? exportDigest,
TargetRepository: resolvedRepository,
ExporterVersion: exporterVersion,
UpdatedAt: now),
UpdatedAt: now,
Files: manifest),
cancellationToken).ConfigureAwait(false);
}
@@ -81,6 +85,7 @@ public sealed class ExportStateManager
TargetRepository = resolvedRepo,
ExporterVersion = exporterVersion,
UpdatedAt = now,
Files = manifest,
}
: existing with
{
@@ -90,6 +95,7 @@ public sealed class ExportStateManager
TargetRepository = resolvedRepo,
ExporterVersion = exporterVersion,
UpdatedAt = now,
Files = manifest,
};
return await _store.UpsertAsync(updatedRecord, cancellationToken).ConfigureAwait(false);
@@ -100,11 +106,13 @@ public sealed class ExportStateManager
string deltaDigest,
string? cursor,
string exporterVersion,
IReadOnlyList<ExportFileRecord> manifest,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrEmpty(exporterId);
ArgumentException.ThrowIfNullOrEmpty(deltaDigest);
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
manifest ??= Array.Empty<ExportFileRecord>();
var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false);
if (existing is null)
@@ -119,6 +127,7 @@ public sealed class ExportStateManager
ExportCursor = cursor ?? existing.ExportCursor,
ExporterVersion = exporterVersion,
UpdatedAt = now,
Files = manifest,
};
return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false);

View File

@@ -9,4 +9,7 @@ public sealed record ExportStateRecord(
string? ExportCursor,
string? TargetRepository,
string? ExporterVersion,
DateTimeOffset UpdatedAt);
DateTimeOffset UpdatedAt,
IReadOnlyList<ExportFileRecord> Files);
public sealed record ExportFileRecord(string Path, long Length, string Digest);

View File

@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
@@ -11,8 +12,7 @@ namespace StellaOps.Feedser.Storage.Mongo;
public sealed class JobRunDocument
{
[BsonId]
[BsonGuidRepresentation(GuidRepresentation.Standard)]
public Guid Id { get; set; }
public string Id { get; set; } = string.Empty;
[BsonElement("kind")]
public string Kind { get; set; } = string.Empty;
@@ -60,7 +60,7 @@ internal static class JobRunDocumentExtensions
{
return new JobRunDocument
{
Id = id,
Id = id.ToString(),
Kind = request.Kind,
Status = JobRunStatus.Pending.ToString(),
Trigger = request.Trigger,
@@ -79,7 +79,7 @@ internal static class JobRunDocumentExtensions
var parameters = document.Parameters?.ToDictionary() ?? new Dictionary<string, object?>();
return new JobRunSnapshot(
document.Id,
Guid.Parse(document.Id),
document.Kind,
Enum.Parse<JobRunStatus>(document.Status, ignoreCase: true),
DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc),

View File

@@ -1,3 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Feedser.Storage.Mongo.MergeEvents;
@@ -6,7 +10,7 @@ namespace StellaOps.Feedser.Storage.Mongo.MergeEvents;
public sealed class MergeEventDocument
{
[BsonId]
public Guid Id { get; set; }
public string Id { get; set; } = string.Empty;
[BsonElement("advisoryKey")]
public string AdvisoryKey { get; set; } = string.Empty;
@@ -21,7 +25,7 @@ public sealed class MergeEventDocument
public DateTime MergedAt { get; set; }
[BsonElement("inputDocuments")]
public List<Guid> InputDocuments { get; set; } = new();
public List<string> InputDocuments { get; set; } = new();
}
internal static class MergeEventDocumentExtensions
@@ -29,20 +33,20 @@ internal static class MergeEventDocumentExtensions
public static MergeEventDocument FromRecord(MergeEventRecord record)
=> new()
{
Id = record.Id,
Id = record.Id.ToString(),
AdvisoryKey = record.AdvisoryKey,
BeforeHash = record.BeforeHash,
AfterHash = record.AfterHash,
MergedAt = record.MergedAt.UtcDateTime,
InputDocuments = record.InputDocumentIds.ToList(),
InputDocuments = record.InputDocumentIds.Select(static id => id.ToString()).ToList(),
};
public static MergeEventRecord ToRecord(this MergeEventDocument document)
=> new(
document.Id,
Guid.Parse(document.Id),
document.AdvisoryKey,
document.BeforeHash,
document.AfterHash,
DateTime.SpecifyKind(document.MergedAt, DateTimeKind.Utc),
document.InputDocuments);
document.InputDocuments.Select(static value => Guid.Parse(value)).ToList());
}

View File

@@ -269,20 +269,22 @@ public sealed class MongoBootstrapper
return collection.Indexes.CreateManyAsync(indexes, cancellationToken);
}
private Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken)
private async Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.PsirtFlags);
var indexes = new List<CreateIndexModel<BsonDocument>>
try
{
new(
Builders<BsonDocument>.IndexKeys.Ascending("advisoryKey"),
new CreateIndexOptions { Name = "psirt_advisoryKey_unique", Unique = true }),
new(
Builders<BsonDocument>.IndexKeys.Ascending("vendor"),
new CreateIndexOptions { Name = "psirt_vendor" }),
};
await collection.Indexes.DropOneAsync("psirt_advisoryKey_unique", cancellationToken).ConfigureAwait(false);
}
catch (MongoCommandException ex) when (ex.CodeName == "IndexNotFound")
{
}
return collection.Indexes.CreateManyAsync(indexes, cancellationToken);
var index = new CreateIndexModel<BsonDocument>(
Builders<BsonDocument>.IndexKeys.Ascending("vendor"),
new CreateIndexOptions { Name = "psirt_vendor" });
await collection.Indexes.CreateOneAsync(index, cancellationToken: cancellationToken).ConfigureAwait(false);
}
private Task EnsureChangeHistoryIndexesAsync(CancellationToken cancellationToken)

View File

@@ -36,7 +36,8 @@ public sealed class MongoJobStore : IJobStore
public async Task<JobRunSnapshot?> TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken)
{
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runId)
var runIdValue = runId.ToString();
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runIdValue)
& Builders<JobRunDocument>.Filter.Eq(x => x.Status, PendingStatus);
var update = Builders<JobRunDocument>.Update
@@ -63,7 +64,8 @@ public sealed class MongoJobStore : IJobStore
public async Task<JobRunSnapshot?> TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken)
{
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runId)
var runIdValue = runId.ToString();
var filter = Builders<JobRunDocument>.Filter.Eq(x => x.Id, runIdValue)
& Builders<JobRunDocument>.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus });
var update = Builders<JobRunDocument>.Update
@@ -91,7 +93,7 @@ public sealed class MongoJobStore : IJobStore
public async Task<JobRunSnapshot?> FindAsync(Guid runId, CancellationToken cancellationToken)
{
var cursor = await _collection.FindAsync(x => x.Id == runId, cancellationToken: cancellationToken).ConfigureAwait(false);
var cursor = await _collection.FindAsync(x => x.Id == runId.ToString(), cancellationToken: cancellationToken).ConfigureAwait(false);
var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
return document?.ToSnapshot();
}

View File

@@ -5,6 +5,7 @@ using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Feedser.Core.Jobs;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Aliases;
using StellaOps.Feedser.Storage.Mongo.ChangeHistory;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
@@ -58,6 +59,7 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IDocumentStore, DocumentStore>();
services.AddSingleton<IDtoStore, DtoStore>();
services.AddSingleton<IAdvisoryStore, AdvisoryStore>();
services.AddSingleton<IAliasStore, AliasStore>();
services.AddSingleton<IChangeHistoryStore, MongoChangeHistoryStore>();
services.AddSingleton<IJpFlagStore, JpFlagStore>();
services.AddSingleton<IPsirtFlagStore, PsirtFlagStore>();

View File

@@ -13,3 +13,4 @@
|Migration playbook for schema/index changes|BE-Storage|Storage.Mongo|DONE `MongoMigrationRunner` executes `IMongoMigration` steps recorded in `schema_migrations`; see `MIGRATIONS.md`.|
|Raw document retention/TTL strategy|BE-Storage|Storage.Mongo|DONE retention options flow into `RawDocumentRetentionService` and TTL migrations for `document`/GridFS indexes.|
|Persist last failure reason in SourceState|BE-Storage|Storage.Mongo|DONE `MongoSourceStateRepository.MarkFailureAsync` stores `lastFailureReason` with length guard + reset on success.|
|AdvisoryStore range primitives deserialization|BE-Storage|Models|DONE BSON helpers handle `RangePrimitives`; regression test covers SemVer/NEVRA/EVR envelopes persisted through Mongo.|