Add authority bootstrap flows and Concelier ops runbooks

This commit is contained in:
2025-10-15 10:03:56 +03:00
parent ea8226120c
commit 0ba025022f
276 changed files with 21674 additions and 934 deletions

View File

@@ -15,7 +15,8 @@ public class StandardClientProvisioningStoreTests
public async Task CreateOrUpdateAsync_HashesSecretAndPersistsDocument()
{
var store = new TrackingClientStore();
var provisioning = new StandardClientProvisioningStore("standard", store);
var revocations = new TrackingRevocationStore();
var provisioning = new StandardClientProvisioningStore("standard", store, revocations, TimeProvider.System);
var registration = new AuthorityClientRegistration(
clientId: "bootstrap-client",
@@ -63,4 +64,21 @@ public class StandardClientProvisioningStoreTests
return ValueTask.FromResult(removed);
}
}
private sealed class TrackingRevocationStore : IAuthorityRevocationStore
{
public List<AuthorityRevocationDocument> Upserts { get; } = new();
public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken)
{
Upserts.Add(document);
return ValueTask.CompletedTask;
}
public ValueTask<bool> RemoveAsync(string category, string revocationId, CancellationToken cancellationToken)
=> ValueTask.FromResult(true);
public ValueTask<IReadOnlyList<AuthorityRevocationDocument>> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
=> ValueTask.FromResult<IReadOnlyList<AuthorityRevocationDocument>>(Array.Empty<AuthorityRevocationDocument>());
}
}

View File

@@ -5,6 +5,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using Mongo2Go;
@@ -58,6 +59,21 @@ public class StandardPluginRegistrarTests
services.AddLogging();
services.AddSingleton<IMongoDatabase>(database);
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
services.AddSingleton(TimeProvider.System);
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
var registrar = new StandardPluginRegistrar();
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
@@ -83,6 +99,53 @@ public class StandardPluginRegistrarTests
Assert.True(verification.User?.RequiresPasswordReset);
}
[Fact]
public void Register_LogsWarning_WhenPasswordPolicyWeaker()
{
using var runner = MongoDbRunner.Start(singleNodeReplSet: true);
var client = new MongoClient(runner.ConnectionString);
var database = client.GetDatabase("registrar-password-policy");
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["passwordPolicy:minimumLength"] = "6",
["passwordPolicy:requireUppercase"] = "false",
["passwordPolicy:requireLowercase"] = "false",
["passwordPolicy:requireDigit"] = "false",
["passwordPolicy:requireSymbol"] = "false"
})
.Build();
var manifest = new AuthorityPluginManifest(
"standard",
"standard",
true,
typeof(StandardPluginRegistrar).Assembly.GetName().Name,
typeof(StandardPluginRegistrar).Assembly.Location,
new[] { AuthorityPluginCapabilities.Password },
new Dictionary<string, string?>(),
"standard.yaml");
var pluginContext = new AuthorityPluginContext(manifest, configuration);
var services = new ServiceCollection();
var loggerProvider = new CapturingLoggerProvider();
services.AddLogging(builder => builder.AddProvider(loggerProvider));
services.AddSingleton<IMongoDatabase>(database);
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
var registrar = new StandardPluginRegistrar();
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
using var provider = services.BuildServiceProvider();
_ = provider.GetRequiredService<StandardUserCredentialStore>();
Assert.Contains(loggerProvider.Entries, entry =>
entry.Level == LogLevel.Warning &&
entry.Category.Contains(typeof(StandardPluginRegistrar).FullName!, StringComparison.Ordinal) &&
entry.Message.Contains("weaker password policy", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Register_ForcesPasswordCapability_WhenManifestMissing()
{
@@ -106,6 +169,8 @@ public class StandardPluginRegistrarTests
services.AddLogging();
services.AddSingleton<IMongoDatabase>(database);
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
services.AddSingleton(TimeProvider.System);
var registrar = new StandardPluginRegistrar();
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
@@ -209,6 +274,61 @@ public class StandardPluginRegistrarTests
}
}
internal sealed record CapturedLogEntry(string Category, LogLevel Level, string Message);
internal sealed class CapturingLoggerProvider : ILoggerProvider
{
public List<CapturedLogEntry> Entries { get; } = new();
public ILogger CreateLogger(string categoryName) => new CapturingLogger(categoryName, Entries);
public void Dispose()
{
}
private sealed class CapturingLogger : ILogger
{
private readonly string category;
private readonly List<CapturedLogEntry> entries;
public CapturingLogger(string category, List<CapturedLogEntry> entries)
{
this.category = category;
this.entries = entries;
}
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
entries.Add(new CapturedLogEntry(category, logLevel, formatter(state, exception)));
}
private sealed class NullScope : IDisposable
{
public static readonly NullScope Instance = new();
public void Dispose()
{
}
}
}
}
internal sealed class StubRevocationStore : IAuthorityRevocationStore
{
public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public ValueTask<bool> RemoveAsync(string category, string revocationId, CancellationToken cancellationToken)
=> ValueTask.FromResult(false);
public ValueTask<IReadOnlyList<AuthorityRevocationDocument>> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
=> ValueTask.FromResult<IReadOnlyList<AuthorityRevocationDocument>>(Array.Empty<AuthorityRevocationDocument>());
}
internal sealed class InMemoryClientStore : IAuthorityClientStore
{
private readonly Dictionary<string, AuthorityClientDocument> clients = new(StringComparer.OrdinalIgnoreCase);

View File

@@ -71,6 +71,41 @@ internal sealed class PasswordPolicyOptions
throw new InvalidOperationException($"Standard plugin '{pluginName}' requires passwordPolicy.minimumLength to be greater than zero.");
}
}
public bool IsWeakerThan(PasswordPolicyOptions other)
{
if (other is null)
{
return false;
}
if (MinimumLength < other.MinimumLength)
{
return true;
}
if (!RequireUppercase && other.RequireUppercase)
{
return true;
}
if (!RequireLowercase && other.RequireLowercase)
{
return true;
}
if (!RequireDigit && other.RequireDigit)
{
return true;
}
if (!RequireSymbol && other.RequireSymbol)
{
return true;
}
return false;
}
}
internal sealed class LockoutOptions

View File

@@ -51,6 +51,25 @@ internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar
var cryptoProvider = sp.GetRequiredService<ICryptoProvider>();
var passwordHasher = new CryptoPasswordHasher(pluginOptions, cryptoProvider);
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
var registrarLogger = loggerFactory.CreateLogger<StandardPluginRegistrar>();
var baselinePolicy = new PasswordPolicyOptions();
if (pluginOptions.PasswordPolicy.IsWeakerThan(baselinePolicy))
{
registrarLogger.LogWarning(
"Standard plugin '{Plugin}' configured a weaker password policy (minLength={Length}, requireUpper={Upper}, requireLower={Lower}, requireDigit={Digit}, requireSymbol={Symbol}) than the baseline (minLength={BaseLength}, requireUpper={BaseUpper}, requireLower={BaseLower}, requireDigit={BaseDigit}, requireSymbol={BaseSymbol}).",
pluginName,
pluginOptions.PasswordPolicy.MinimumLength,
pluginOptions.PasswordPolicy.RequireUppercase,
pluginOptions.PasswordPolicy.RequireLowercase,
pluginOptions.PasswordPolicy.RequireDigit,
pluginOptions.PasswordPolicy.RequireSymbol,
baselinePolicy.MinimumLength,
baselinePolicy.RequireUppercase,
baselinePolicy.RequireLowercase,
baselinePolicy.RequireDigit,
baselinePolicy.RequireSymbol);
}
return new StandardUserCredentialStore(
pluginName,

View File

@@ -5,12 +5,14 @@
| PLG6.DOC | DONE (2025-10-11) | BE-Auth Plugin, Docs Guild | PLG1PLG5 | Final polish + diagrams for plugin developer guide (AUTHPLUG-DOCS-01-001). | Docs team delivers copy-edit + exported diagrams; PR merged. |
| SEC1.PLG | DONE (2025-10-11) | Security Guild, BE-Auth Plugin | SEC1.A (StellaOps.Cryptography) | Swap Standard plugin hashing to Argon2id via `StellaOps.Cryptography` abstractions; keep PBKDF2 verification for legacy. | ✅ `StandardUserCredentialStore` uses `ICryptoProvider` to hash/check; ✅ Transparent rehash on success; ✅ Unit tests cover tamper + legacy rehash. |
| SEC1.OPT | DONE (2025-10-11) | Security Guild | SEC1.PLG | Expose password hashing knobs in `StandardPluginOptions` (`memoryKiB`, `iterations`, `parallelism`, `algorithm`) with validation. | ✅ Options bound from YAML; ✅ Invalid configs throw; ✅ Docs include tuning guidance. |
| SEC2.PLG | TODO | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. |
| SEC3.PLG | TODO | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after). | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. |
| SEC2.PLG | DOING (2025-10-14) | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. |
| SEC3.PLG | DOING (2025-10-14) | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after). | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. |
| SEC4.PLG | DONE (2025-10-12) | Security Guild | SEC4.A (revocation schema) | Provide plugin hooks so revoked users/clients write reasons for revocation bundle export. | ✅ Revocation exporter consumes plugin data; ✅ Tests cover revoked user/client output. |
| SEC5.PLG | TODO | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. |
| SEC5.PLG | DOING (2025-10-14) | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. |
| PLG4-6.CAPABILITIES | BLOCKED (2025-10-12) | BE-Auth Plugin, Docs Guild | PLG1PLG3 | Finalise capability metadata exposure, config validation, and developer guide updates; remaining action is Docs polish/diagram export. | ✅ Capability metadata + validation merged; ✅ Plugin guide updated with final copy & diagrams; ✅ Release notes mention new toggles. <br>⛔ Blocked awaiting Authority rate-limiter stream (CORE8/SEC3) to resume so doc updates reflect final limiter behaviour. |
| PLG7.RFC | REVIEW | BE-Auth Plugin, Security Guild | PLG4 | Socialize LDAP plugin RFC (`docs/rfcs/authority-plugin-ldap.md`) and capture guild feedback. | ✅ Guild review sign-off recorded; ✅ Follow-up issues filed in module boards. |
| PLG6.DIAGRAM | TODO | Docs Guild | PLG6.DOC | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | ✅ Mermaid sources committed; ✅ Rendered SVG/PNG linked from Section 2 + Section 9; ✅ Docs build preview shared with Plugin + Docs guilds. |
> Update statuses to DOING/DONE/BLOCKED as you make progress. Always run `dotnet test` for touched projects before marking DONE.
> Remark (2025-10-13, PLG6.DOC/PLG6.DIAGRAM): Security Guild delivered `docs/security/rate-limits.md`; Docs team can lift Section 3 (tuning table + alerts) into the developer guide diagrams when rendering assets.

View File

@@ -22,5 +22,6 @@ public static class AuthorityMongoDefaults
public const string LoginAttempts = "authority_login_attempts";
public const string Revocations = "authority_revocations";
public const string RevocationState = "authority_revocation_state";
public const string Invites = "authority_bootstrap_invites";
}
}

View File

@@ -0,0 +1,72 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Authority.Storage.Mongo.Documents;
/// <summary>
/// Represents a bootstrap invitation token for provisioning users or clients.
/// </summary>
[BsonIgnoreExtraElements]
public sealed class AuthorityBootstrapInviteDocument
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string Id { get; set; } = ObjectId.GenerateNewId().ToString();
[BsonElement("token")]
public string Token { get; set; } = Guid.NewGuid().ToString("N");
[BsonElement("type")]
public string Type { get; set; } = "user";
[BsonElement("provider")]
[BsonIgnoreIfNull]
public string? Provider { get; set; }
[BsonElement("target")]
[BsonIgnoreIfNull]
public string? Target { get; set; }
[BsonElement("issuedAt")]
public DateTimeOffset IssuedAt { get; set; } = DateTimeOffset.UtcNow;
[BsonElement("issuedBy")]
[BsonIgnoreIfNull]
public string? IssuedBy { get; set; }
[BsonElement("expiresAt")]
public DateTimeOffset ExpiresAt { get; set; } = DateTimeOffset.UtcNow.AddDays(2);
[BsonElement("status")]
public string Status { get; set; } = AuthorityBootstrapInviteStatuses.Pending;
[BsonElement("reservedAt")]
[BsonIgnoreIfNull]
public DateTimeOffset? ReservedAt { get; set; }
[BsonElement("reservedBy")]
[BsonIgnoreIfNull]
public string? ReservedBy { get; set; }
[BsonElement("consumedAt")]
[BsonIgnoreIfNull]
public DateTimeOffset? ConsumedAt { get; set; }
[BsonElement("consumedBy")]
[BsonIgnoreIfNull]
public string? ConsumedBy { get; set; }
[BsonElement("metadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
}
public static class AuthorityBootstrapInviteStatuses
{
public const string Pending = "pending";
public const string Reserved = "reserved";
public const string Consumed = "consumed";
public const string Expired = "expired";
}

View File

@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
@@ -61,6 +62,11 @@ public sealed class AuthorityTokenDocument
[BsonIgnoreIfNull]
public string? RevokedReasonDescription { get; set; }
[BsonElement("devices")]
[BsonIgnoreIfNull]
public List<BsonDocument>? Devices { get; set; }
[BsonElement("revokedMetadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? RevokedMetadata { get; set; }

View File

@@ -98,12 +98,19 @@ public static class ServiceCollectionExtensions
return database.GetCollection<AuthorityRevocationExportStateDocument>(AuthorityMongoDefaults.Collections.RevocationState);
});
services.AddSingleton(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
return database.GetCollection<AuthorityBootstrapInviteDocument>(AuthorityMongoDefaults.Collections.Invites);
});
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityUserCollectionInitializer>();
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityClientCollectionInitializer>();
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityScopeCollectionInitializer>();
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityTokenCollectionInitializer>();
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityLoginAttemptCollectionInitializer>();
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityRevocationCollectionInitializer>();
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityBootstrapInviteCollectionInitializer>();
services.TryAddSingleton<IAuthorityUserStore, AuthorityUserStore>();
services.TryAddSingleton<IAuthorityClientStore, AuthorityClientStore>();
@@ -112,6 +119,7 @@ public static class ServiceCollectionExtensions
services.TryAddSingleton<IAuthorityLoginAttemptStore, AuthorityLoginAttemptStore>();
services.TryAddSingleton<IAuthorityRevocationStore, AuthorityRevocationStore>();
services.TryAddSingleton<IAuthorityRevocationExportStateStore, AuthorityRevocationExportStateStore>();
services.TryAddSingleton<IAuthorityBootstrapInviteStore, AuthorityBootstrapInviteStore>();
return services;
}

View File

@@ -0,0 +1,25 @@
using MongoDB.Driver;
using StellaOps.Authority.Storage.Mongo.Documents;
namespace StellaOps.Authority.Storage.Mongo.Initialization;
internal sealed class AuthorityBootstrapInviteCollectionInitializer : IAuthorityCollectionInitializer
{
private static readonly CreateIndexModel<AuthorityBootstrapInviteDocument>[] Indexes =
{
new CreateIndexModel<AuthorityBootstrapInviteDocument>(
Builders<AuthorityBootstrapInviteDocument>.IndexKeys.Ascending(i => i.Token),
new CreateIndexOptions { Unique = true, Name = "idx_invite_token" }),
new CreateIndexModel<AuthorityBootstrapInviteDocument>(
Builders<AuthorityBootstrapInviteDocument>.IndexKeys.Ascending(i => i.Status).Ascending(i => i.ExpiresAt),
new CreateIndexOptions { Name = "idx_invite_status_expires" })
};
public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
var collection = database.GetCollection<AuthorityBootstrapInviteDocument>(AuthorityMongoDefaults.Collections.Invites);
await collection.Indexes.CreateManyAsync(Indexes, cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,166 @@
using System;
using System.Collections.Generic;
using MongoDB.Driver;
using StellaOps.Authority.Storage.Mongo.Documents;
namespace StellaOps.Authority.Storage.Mongo.Stores;
internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteStore
{
private readonly IMongoCollection<AuthorityBootstrapInviteDocument> collection;
public AuthorityBootstrapInviteStore(IMongoCollection<AuthorityBootstrapInviteDocument> collection)
=> this.collection = collection ?? throw new ArgumentNullException(nameof(collection));
public async ValueTask<AuthorityBootstrapInviteDocument> CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
return document;
}
public async ValueTask<BootstrapInviteReservationResult> TryReserveAsync(
string token,
string expectedType,
DateTimeOffset now,
string? reservedBy,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(token))
{
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null);
}
var normalizedToken = token.Trim();
var filter = Builders<AuthorityBootstrapInviteDocument>.Filter.And(
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, normalizedToken),
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Pending));
var update = Builders<AuthorityBootstrapInviteDocument>.Update
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)
.Set(i => i.ReservedAt, now)
.Set(i => i.ReservedBy, reservedBy);
var options = new FindOneAndUpdateOptions<AuthorityBootstrapInviteDocument>
{
ReturnDocument = ReturnDocument.After
};
var invite = await collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false);
if (invite is null)
{
var existing = await collection
.Find(i => i.Token == normalizedToken)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (existing is null)
{
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null);
}
if (existing.Status is AuthorityBootstrapInviteStatuses.Consumed or AuthorityBootstrapInviteStatuses.Reserved)
{
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.AlreadyUsed, existing);
}
if (existing.Status == AuthorityBootstrapInviteStatuses.Expired || existing.ExpiresAt <= now)
{
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Expired, existing);
}
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, existing);
}
if (!string.Equals(invite.Type, expectedType, StringComparison.OrdinalIgnoreCase))
{
await ReleaseAsync(normalizedToken, cancellationToken).ConfigureAwait(false);
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, invite);
}
if (invite.ExpiresAt <= now)
{
await MarkExpiredAsync(normalizedToken, cancellationToken).ConfigureAwait(false);
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Expired, invite);
}
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Reserved, invite);
}
public async ValueTask<bool> ReleaseAsync(string token, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(token))
{
return false;
}
var result = await collection.UpdateOneAsync(
Builders<AuthorityBootstrapInviteDocument>.Filter.And(
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, token.Trim()),
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)),
Builders<AuthorityBootstrapInviteDocument>.Update
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Pending)
.Set(i => i.ReservedAt, null)
.Set(i => i.ReservedBy, null),
cancellationToken: cancellationToken).ConfigureAwait(false);
return result.ModifiedCount > 0;
}
public async ValueTask<bool> MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(token))
{
return false;
}
var result = await collection.UpdateOneAsync(
Builders<AuthorityBootstrapInviteDocument>.Filter.And(
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, token.Trim()),
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)),
Builders<AuthorityBootstrapInviteDocument>.Update
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Consumed)
.Set(i => i.ConsumedAt, consumedAt)
.Set(i => i.ConsumedBy, consumedBy),
cancellationToken: cancellationToken).ConfigureAwait(false);
return result.ModifiedCount > 0;
}
public async ValueTask<IReadOnlyList<AuthorityBootstrapInviteDocument>> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var filter = Builders<AuthorityBootstrapInviteDocument>.Filter.And(
Builders<AuthorityBootstrapInviteDocument>.Filter.Lte(i => i.ExpiresAt, now),
Builders<AuthorityBootstrapInviteDocument>.Filter.In(
i => i.Status,
new[] { AuthorityBootstrapInviteStatuses.Pending, AuthorityBootstrapInviteStatuses.Reserved }));
var update = Builders<AuthorityBootstrapInviteDocument>.Update
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Expired)
.Set(i => i.ReservedAt, null)
.Set(i => i.ReservedBy, null);
var expired = await collection.Find(filter)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
if (expired.Count == 0)
{
return Array.Empty<AuthorityBootstrapInviteDocument>();
}
await collection.UpdateManyAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false);
return expired;
}
private async Task MarkExpiredAsync(string token, CancellationToken cancellationToken)
{
await collection.UpdateOneAsync(
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, token),
Builders<AuthorityBootstrapInviteDocument>.Update.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Expired),
cancellationToken: cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -1,6 +1,10 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using System.Linq;
using System.Globalization;
using StellaOps.Authority.Storage.Mongo.Documents;
namespace StellaOps.Authority.Storage.Mongo.Stores;
@@ -86,6 +90,86 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore
logger.LogDebug("Updated token {TokenId} status to {Status} (matched {Matched}).", tokenId, status, result.MatchedCount);
}
public async ValueTask<TokenUsageUpdateResult> RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(tokenId))
{
return new TokenUsageUpdateResult(TokenUsageUpdateStatus.NotFound, null, null);
}
if (string.IsNullOrWhiteSpace(remoteAddress) && string.IsNullOrWhiteSpace(userAgent))
{
return new TokenUsageUpdateResult(TokenUsageUpdateStatus.MissingMetadata, remoteAddress, userAgent);
}
var id = tokenId.Trim();
var token = await collection
.Find(t => t.TokenId == id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (token is null)
{
return new TokenUsageUpdateResult(TokenUsageUpdateStatus.NotFound, remoteAddress, userAgent);
}
token.Devices ??= new List<BsonDocument>();
string? normalizedAddress = string.IsNullOrWhiteSpace(remoteAddress) ? null : remoteAddress.Trim();
string? normalizedAgent = string.IsNullOrWhiteSpace(userAgent) ? null : userAgent.Trim();
var device = token.Devices.FirstOrDefault(d =>
string.Equals(GetString(d, "remoteAddress"), normalizedAddress, StringComparison.OrdinalIgnoreCase) &&
string.Equals(GetString(d, "userAgent"), normalizedAgent, StringComparison.Ordinal));
var suspicious = false;
if (device is null)
{
suspicious = token.Devices.Count > 0;
var document = new BsonDocument
{
{ "remoteAddress", normalizedAddress },
{ "userAgent", normalizedAgent },
{ "firstSeen", BsonDateTime.Create(observedAt.UtcDateTime) },
{ "lastSeen", BsonDateTime.Create(observedAt.UtcDateTime) },
{ "useCount", 1 }
};
token.Devices.Add(document);
}
else
{
device["lastSeen"] = BsonDateTime.Create(observedAt.UtcDateTime);
device["useCount"] = device.TryGetValue("useCount", out var existingCount) && existingCount.IsInt32
? existingCount.AsInt32 + 1
: 1;
}
var update = Builders<AuthorityTokenDocument>.Update.Set(t => t.Devices, token.Devices);
await collection.UpdateOneAsync(
Builders<AuthorityTokenDocument>.Filter.Eq(t => t.TokenId, id),
update,
cancellationToken: cancellationToken).ConfigureAwait(false);
return new TokenUsageUpdateResult(suspicious ? TokenUsageUpdateStatus.SuspectedReplay : TokenUsageUpdateStatus.Recorded, normalizedAddress, normalizedAgent);
}
private static string? GetString(BsonDocument document, string name)
{
if (!document.TryGetValue(name, out var value))
{
return null;
}
return value switch
{
{ IsString: true } => value.AsString,
{ IsBsonNull: true } => null,
_ => value.ToString()
};
}
public async ValueTask<long> DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken)
{
var filter = Builders<AuthorityTokenDocument>.Filter.And(

View File

@@ -0,0 +1,26 @@
using StellaOps.Authority.Storage.Mongo.Documents;
namespace StellaOps.Authority.Storage.Mongo.Stores;
public interface IAuthorityBootstrapInviteStore
{
ValueTask<AuthorityBootstrapInviteDocument> CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken);
ValueTask<BootstrapInviteReservationResult> TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken);
ValueTask<bool> ReleaseAsync(string token, CancellationToken cancellationToken);
ValueTask<bool> MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken);
ValueTask<IReadOnlyList<AuthorityBootstrapInviteDocument>> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken);
}
public enum BootstrapInviteReservationStatus
{
Reserved,
NotFound,
Expired,
AlreadyUsed
}
public sealed record BootstrapInviteReservationResult(BootstrapInviteReservationStatus Status, AuthorityBootstrapInviteDocument? Invite);

View File

@@ -1,3 +1,5 @@
using System;
using System.Collections.Generic;
using StellaOps.Authority.Storage.Mongo.Documents;
namespace StellaOps.Authority.Storage.Mongo.Stores;
@@ -21,5 +23,17 @@ public interface IAuthorityTokenStore
ValueTask<long> DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken);
ValueTask<TokenUsageUpdateResult> RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken);
ValueTask<IReadOnlyList<AuthorityTokenDocument>> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken);
}
public enum TokenUsageUpdateStatus
{
Recorded,
SuspectedReplay,
MissingMetadata,
NotFound
}
public sealed record TokenUsageUpdateResult(TokenUsageUpdateStatus Status, string? RemoteAddress, string? UserAgent);

View File

@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Authority.Bootstrap;
using StellaOps.Authority.Storage.Mongo.Documents;
using StellaOps.Authority.Storage.Mongo.Stores;
using StellaOps.Cryptography.Audit;
using Xunit;
namespace StellaOps.Authority.Tests.Bootstrap;
public sealed class BootstrapInviteCleanupServiceTests
{
[Fact]
public async Task SweepExpiredInvitesAsync_ExpiresInvitesAndEmitsAuditRecords()
{
var now = new DateTimeOffset(2025, 10, 14, 12, 0, 0, TimeSpan.Zero);
var timeProvider = new FakeTimeProvider(now);
var invites = new List<AuthorityBootstrapInviteDocument>
{
new()
{
Token = "token-1",
Type = BootstrapInviteTypes.User,
ExpiresAt = now.AddMinutes(-5),
Provider = "standard",
Target = "alice@example.com",
Status = AuthorityBootstrapInviteStatuses.Pending
},
new()
{
Token = "token-2",
Type = BootstrapInviteTypes.Client,
ExpiresAt = now.AddMinutes(-1),
Provider = "standard",
Target = "client-1",
Status = AuthorityBootstrapInviteStatuses.Reserved
}
};
var store = new FakeInviteStore(invites);
var sink = new CapturingAuthEventSink();
var service = new BootstrapInviteCleanupService(store, sink, timeProvider, NullLogger<BootstrapInviteCleanupService>.Instance);
await service.SweepExpiredInvitesAsync(CancellationToken.None);
Assert.True(store.ExpireCalled);
Assert.Equal(2, sink.Events.Count);
Assert.All(sink.Events, record => Assert.Equal("authority.bootstrap.invite.expired", record.EventType));
Assert.Contains(sink.Events, record => record.Properties.Any(property => property.Name == "invite.token" && property.Value.Value == "token-1"));
Assert.Contains(sink.Events, record => record.Properties.Any(property => property.Name == "invite.token" && property.Value.Value == "token-2"));
}
private sealed class FakeInviteStore : IAuthorityBootstrapInviteStore
{
private readonly IReadOnlyList<AuthorityBootstrapInviteDocument> invites;
public FakeInviteStore(IReadOnlyList<AuthorityBootstrapInviteDocument> invites)
=> this.invites = invites;
public bool ExpireCalled { get; private set; }
public ValueTask<AuthorityBootstrapInviteDocument> CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken)
=> throw new NotImplementedException();
public ValueTask<BootstrapInviteReservationResult> TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken)
=> ValueTask.FromResult(new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null));
public ValueTask<bool> ReleaseAsync(string token, CancellationToken cancellationToken)
=> ValueTask.FromResult(false);
public ValueTask<bool> MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken)
=> ValueTask.FromResult(false);
public ValueTask<IReadOnlyList<AuthorityBootstrapInviteDocument>> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
ExpireCalled = true;
return ValueTask.FromResult(invites);
}
}
private sealed class CapturingAuthEventSink : IAuthEventSink
{
public List<AuthEventRecord> Events { get; } = new();
public ValueTask WriteAsync(AuthEventRecord record, CancellationToken cancellationToken)
{
Events.Add(record);
return ValueTask.CompletedTask;
}
}
}

View File

@@ -17,6 +17,7 @@ using StellaOps.Authority.Storage.Mongo.Stores;
using StellaOps.Authority.RateLimiting;
using StellaOps.Cryptography.Audit;
using Xunit;
using MongoDB.Bson;
using static StellaOps.Authority.Tests.OpenIddict.TestHelpers;
namespace StellaOps.Authority.Tests.OpenIddict;
@@ -76,7 +77,7 @@ public class ClientCredentialsHandlersTests
await handler.HandleAsync(context);
Assert.False(context.IsRejected);
Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}");
Assert.Same(clientDocument, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty]);
var grantedScopes = Assert.IsType<string[]>(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]);
@@ -84,6 +85,36 @@ public class ClientCredentialsHandlersTests
Assert.Equal(clientDocument.Plugin, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty]);
}
[Fact]
public async Task ValidateClientCredentials_EmitsTamperAuditEvent_WhenUnexpectedParametersPresent()
{
var clientDocument = CreateClient(
secret: "s3cr3t!",
allowedGrantTypes: "client_credentials",
allowedScopes: "jobs:read");
var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument));
var sink = new TestAuthEventSink();
var handler = new ValidateClientCredentialsHandler(
new TestClientStore(clientDocument),
registry,
TestActivitySource,
sink,
new TestRateLimiterMetadataAccessor(),
TimeProvider.System,
NullLogger<ValidateClientCredentialsHandler>.Instance);
var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read");
transaction.Request?.SetParameter("unexpected_param", "value");
await handler.HandleAsync(new OpenIddictServerEvents.ValidateTokenRequestContext(transaction));
var tamperEvent = Assert.Single(sink.Events, record => record.EventType == "authority.token.tamper");
Assert.Contains(tamperEvent.Properties, property =>
string.Equals(property.Name, "request.unexpected_parameter", StringComparison.OrdinalIgnoreCase) &&
string.Equals(property.Value.Value, "unexpected_param", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public async Task HandleClientCredentials_PersistsTokenAndEnrichesClaims()
{
@@ -98,22 +129,30 @@ public class ClientCredentialsHandlersTests
var tokenStore = new TestTokenStore();
var authSink = new TestAuthEventSink();
var metadataAccessor = new TestRateLimiterMetadataAccessor();
var validateHandler = new ValidateClientCredentialsHandler(
new TestClientStore(clientDocument),
registry,
TestActivitySource,
authSink,
metadataAccessor,
TimeProvider.System,
NullLogger<ValidateClientCredentialsHandler>.Instance);
var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger");
transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(30);
var validateContext = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction);
await validateHandler.HandleAsync(validateContext);
Assert.False(validateContext.IsRejected);
var handler = new HandleClientCredentialsHandler(
registry,
tokenStore,
TimeProvider.System,
TestActivitySource,
authSink,
metadataAccessor,
NullLogger<HandleClientCredentialsHandler>.Instance);
var persistHandler = new PersistTokensHandler(tokenStore, TimeProvider.System, TestActivitySource, NullLogger<PersistTokensHandler>.Instance);
var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger");
transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(30);
transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = clientDocument;
transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = clientDocument.Plugin!;
transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = new[] { "jobs:trigger" };
var context = new OpenIddictServerEvents.HandleTokenRequestContext(transaction);
await handler.HandleAsync(context);
@@ -161,10 +200,14 @@ public class TokenValidationHandlersTests
ClientId = "feedser"
};
var metadataAccessor = new TestRateLimiterMetadataAccessor();
var auditSink = new TestAuthEventSink();
var handler = new ValidateAccessTokenHandler(
tokenStore,
new TestClientStore(CreateClient()),
CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(CreateClient())),
metadataAccessor,
auditSink,
TimeProvider.System,
TestActivitySource,
NullLogger<ValidateAccessTokenHandler>.Instance);
@@ -203,10 +246,14 @@ public class TokenValidationHandlersTests
var registry = new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger<AuthorityIdentityProviderRegistry>.Instance);
var metadataAccessorSuccess = new TestRateLimiterMetadataAccessor();
var auditSinkSuccess = new TestAuthEventSink();
var handler = new ValidateAccessTokenHandler(
new TestTokenStore(),
new TestClientStore(clientDocument),
registry,
metadataAccessorSuccess,
auditSinkSuccess,
TimeProvider.System,
TestActivitySource,
NullLogger<ValidateAccessTokenHandler>.Instance);
@@ -229,6 +276,76 @@ public class TokenValidationHandlersTests
Assert.False(context.IsRejected);
Assert.Contains(principal.Claims, claim => claim.Type == "enriched" && claim.Value == "true");
}
[Fact]
public async Task ValidateAccessTokenHandler_EmitsReplayAudit_WhenStoreDetectsSuspectedReplay()
{
var tokenStore = new TestTokenStore();
tokenStore.Inserted = new AuthorityTokenDocument
{
TokenId = "token-replay",
Status = "valid",
ClientId = "agent",
Devices = new List<BsonDocument>
{
new BsonDocument
{
{ "remoteAddress", "10.0.0.1" },
{ "userAgent", "agent/1.0" },
{ "firstSeen", BsonDateTime.Create(DateTimeOffset.UtcNow.AddMinutes(-15)) },
{ "lastSeen", BsonDateTime.Create(DateTimeOffset.UtcNow.AddMinutes(-5)) },
{ "useCount", 2 }
}
}
};
tokenStore.UsageCallback = (remote, agent) => new TokenUsageUpdateResult(TokenUsageUpdateStatus.SuspectedReplay, remote, agent);
var metadataAccessor = new TestRateLimiterMetadataAccessor();
var metadata = metadataAccessor.GetMetadata();
if (metadata is not null)
{
metadata.RemoteIp = "203.0.113.7";
metadata.UserAgent = "agent/2.0";
}
var clientDocument = CreateClient();
clientDocument.ClientId = "agent";
var auditSink = new TestAuthEventSink();
var registry = CreateRegistry(withClientProvisioning: false, clientDescriptor: null);
var handler = new ValidateAccessTokenHandler(
tokenStore,
new TestClientStore(clientDocument),
registry,
metadataAccessor,
auditSink,
TimeProvider.System,
TestActivitySource,
NullLogger<ValidateAccessTokenHandler>.Instance);
var transaction = new OpenIddictServerTransaction
{
Options = new OpenIddictServerOptions(),
EndpointType = OpenIddictServerEndpointType.Introspection,
Request = new OpenIddictRequest()
};
var principal = CreatePrincipal("agent", "token-replay", "standard");
var context = new OpenIddictServerEvents.ValidateTokenContext(transaction)
{
Principal = principal,
TokenId = "token-replay"
};
await handler.HandleAsync(context);
Assert.False(context.IsRejected);
var replayEvent = Assert.Single(auditSink.Events, record => record.EventType == "authority.token.replay.suspected");
Assert.Equal(AuthEventOutcome.Error, replayEvent.Outcome);
Assert.NotNull(replayEvent.Network);
Assert.Equal("203.0.113.7", replayEvent.Network?.RemoteAddress.Value);
Assert.Contains(replayEvent.Properties, property => property.Name == "token.devices.total");
}
}
internal sealed class TestClientStore : IAuthorityClientStore
@@ -263,6 +380,8 @@ internal sealed class TestTokenStore : IAuthorityTokenStore
{
public AuthorityTokenDocument? Inserted { get; set; }
public Func<string?, string?, TokenUsageUpdateResult>? UsageCallback { get; set; }
public ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken)
{
Inserted = document;
@@ -281,6 +400,9 @@ internal sealed class TestTokenStore : IAuthorityTokenStore
public ValueTask<long> DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken)
=> ValueTask.FromResult(0L);
public ValueTask<TokenUsageUpdateResult> RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken)
=> ValueTask.FromResult(UsageCallback?.Invoke(remoteAddress, userAgent) ?? new TokenUsageUpdateResult(TokenUsageUpdateStatus.Recorded, remoteAddress, userAgent));
public ValueTask<IReadOnlyList<AuthorityTokenDocument>> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken)
=> ValueTask.FromResult<IReadOnlyList<AuthorityTokenDocument>>(Array.Empty<AuthorityTokenDocument>());
}

View File

@@ -74,6 +74,26 @@ public class PasswordGrantHandlersTests
Assert.Contains(sink.Events, record => record.EventType == "authority.password.grant" && record.Outcome == AuthEventOutcome.LockedOut);
}
[Fact]
public async Task ValidatePasswordGrant_EmitsTamperAuditEvent_WhenUnexpectedParametersPresent()
{
var sink = new TestAuthEventSink();
var metadataAccessor = new TestRateLimiterMetadataAccessor();
var registry = CreateRegistry(new SuccessCredentialStore());
var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger<ValidatePasswordGrantHandler>.Instance);
var transaction = CreatePasswordTransaction("alice", "Password1!");
transaction.Request?.SetParameter("unexpected_param", "value");
await validate.HandleAsync(new OpenIddictServerEvents.ValidateTokenRequestContext(transaction));
var tamperEvent = Assert.Single(sink.Events, record => record.EventType == "authority.token.tamper");
Assert.Equal(AuthEventOutcome.Failure, tamperEvent.Outcome);
Assert.Contains(tamperEvent.Properties, property =>
string.Equals(property.Name, "request.unexpected_parameter", StringComparison.OrdinalIgnoreCase) &&
string.Equals(property.Value.Value, "unexpected_param", StringComparison.OrdinalIgnoreCase));
}
private static AuthorityIdentityProviderRegistry CreateRegistry(IUserCredentialStore store)
{
var plugin = new StubIdentityProviderPlugin("stub", store);
@@ -104,14 +124,14 @@ public class PasswordGrantHandlersTests
Name = name;
Type = "stub";
var manifest = new AuthorityPluginManifest(
name,
"stub",
enabled: true,
version: null,
description: null,
capabilities: new[] { AuthorityPluginCapabilities.Password },
configuration: new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase),
configPath: $"{name}.yaml");
Name: name,
Type: "stub",
Enabled: true,
AssemblyName: null,
AssemblyPath: null,
Capabilities: new[] { AuthorityPluginCapabilities.Password },
Metadata: new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase),
ConfigPath: $"{name}.yaml");
Context = new AuthorityPluginContext(manifest, new ConfigurationBuilder().Build());
Credentials = store;
ClaimsEnricher = new NoopClaimsEnricher();

View File

@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using MongoDB.Driver;
using MongoDB.Bson;
using OpenIddict.Abstractions;
using OpenIddict.Server;
using StellaOps.Authority;
@@ -56,10 +57,10 @@ public sealed class TokenPersistenceIntegrationTests
withClientProvisioning: true,
clientDescriptor: TestHelpers.CreateDescriptor(clientDocument));
var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, NullLogger<ValidateClientCredentialsHandler>.Instance);
var authSink = new TestAuthEventSink();
var metadataAccessor = new TestRateLimiterMetadataAccessor();
var handleHandler = new HandleClientCredentialsHandler(registry, TestActivitySource, authSink, metadataAccessor, clock, NullLogger<HandleClientCredentialsHandler>.Instance);
var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, NullLogger<ValidateClientCredentialsHandler>.Instance);
var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, clock, TestActivitySource, NullLogger<HandleClientCredentialsHandler>.Instance);
var persistHandler = new PersistTokensHandler(tokenStore, clock, TestActivitySource, NullLogger<PersistTokensHandler>.Instance);
var transaction = TestHelpers.CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:trigger");
@@ -148,10 +149,14 @@ public sealed class TokenPersistenceIntegrationTests
var revokedAt = now.AddMinutes(1);
await tokenStore.UpdateStatusAsync(revokedTokenId, "revoked", revokedAt, "manual", null, null, CancellationToken.None);
var metadataAccessor = new TestRateLimiterMetadataAccessor();
var auditSink = new TestAuthEventSink();
var handler = new ValidateAccessTokenHandler(
tokenStore,
clientStore,
registry,
metadataAccessor,
auditSink,
clock,
TestActivitySource,
NullLogger<ValidateAccessTokenHandler>.Instance);
@@ -190,6 +195,60 @@ public sealed class TokenPersistenceIntegrationTests
Assert.Equal("manual", stored.RevokedReason);
}
[Fact]
public async Task RecordUsageAsync_FlagsSuspectedReplay_OnNewDeviceFingerprint()
{
await ResetCollectionsAsync();
var issuedAt = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero);
var clock = new FakeTimeProvider(issuedAt);
await using var provider = await BuildMongoProviderAsync(clock);
var tokenStore = provider.GetRequiredService<IAuthorityTokenStore>();
var tokenDocument = new AuthorityTokenDocument
{
TokenId = "token-replay",
Type = OpenIddictConstants.TokenTypeHints.AccessToken,
ClientId = "client-1",
Status = "valid",
CreatedAt = issuedAt,
Devices = new List<BsonDocument>
{
new BsonDocument
{
{ "remoteAddress", "10.0.0.1" },
{ "userAgent", "agent/1.0" },
{ "firstSeen", BsonDateTime.Create(issuedAt.AddMinutes(-10).UtcDateTime) },
{ "lastSeen", BsonDateTime.Create(issuedAt.AddMinutes(-5).UtcDateTime) },
{ "useCount", 2 }
}
}
};
await tokenStore.InsertAsync(tokenDocument, CancellationToken.None);
var result = await tokenStore.RecordUsageAsync(
"token-replay",
remoteAddress: "10.0.0.2",
userAgent: "agent/2.0",
observedAt: clock.GetUtcNow(),
CancellationToken.None);
Assert.Equal(TokenUsageUpdateStatus.SuspectedReplay, result.Status);
var stored = await tokenStore.FindByTokenIdAsync("token-replay", CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(2, stored!.Devices?.Count);
Assert.Contains(stored.Devices!, doc =>
{
var remote = doc.TryGetValue("remoteAddress", out var ra) && ra.IsString ? ra.AsString : null;
var agentValue = doc.TryGetValue("userAgent", out var ua) && ua.IsString ? ua.AsString : null;
return remote == "10.0.0.2" && agentValue == "agent/2.0";
});
}
private async Task ResetCollectionsAsync()
{
var tokens = fixture.Database.GetCollection<AuthorityTokenDocument>(AuthorityMongoDefaults.Collections.Tokens);
@@ -220,27 +279,3 @@ public sealed class TokenPersistenceIntegrationTests
return provider;
}
}
internal sealed class TestAuthEventSink : IAuthEventSink
{
public List<AuthEventRecord> Records { get; } = new();
public ValueTask WriteAsync(AuthEventRecord record, CancellationToken cancellationToken)
{
Records.Add(record);
return ValueTask.CompletedTask;
}
}
internal sealed class TestRateLimiterMetadataAccessor : IAuthorityRateLimiterMetadataAccessor
{
private readonly AuthorityRateLimiterMetadata metadata = new();
public AuthorityRateLimiterMetadata? GetMetadata() => metadata;
public void SetClientId(string? clientId) => metadata.ClientId = string.IsNullOrWhiteSpace(clientId) ? null : clientId;
public void SetSubjectId(string? subjectId) => metadata.SubjectId = string.IsNullOrWhiteSpace(subjectId) ? null : subjectId;
public void SetTag(string name, string? value) => metadata.SetTag(name, value);
}

View File

@@ -76,6 +76,7 @@ public class AuthorityRateLimiterMetadataMiddlewareTests
context.Request.Path = "/token";
context.Request.Method = HttpMethods.Post;
context.Request.Headers["X-Forwarded-For"] = "203.0.113.99";
context.Request.Headers.UserAgent = "StellaOps-Client/1.2";
var middleware = CreateMiddleware();
await middleware.InvokeAsync(context);
@@ -84,6 +85,9 @@ public class AuthorityRateLimiterMetadataMiddlewareTests
Assert.NotNull(metadata);
Assert.Equal("203.0.113.99", metadata!.RemoteIp);
Assert.Equal("203.0.113.99", metadata.ForwardedFor);
Assert.Equal("StellaOps-Client/1.2", metadata.UserAgent);
Assert.True(metadata.Tags.TryGetValue("authority.user_agent", out var tagValue));
Assert.Equal("StellaOps-Client/1.2", tagValue);
}
private static AuthorityRateLimiterMetadataMiddleware CreateMiddleware()

View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Authority.Storage.Mongo.Stores;
using StellaOps.Authority.Storage.Mongo.Documents;
using StellaOps.Cryptography.Audit;
namespace StellaOps.Authority.Bootstrap;
internal sealed class BootstrapInviteCleanupService : BackgroundService
{
private readonly IAuthorityBootstrapInviteStore inviteStore;
private readonly IAuthEventSink auditSink;
private readonly TimeProvider timeProvider;
private readonly ILogger<BootstrapInviteCleanupService> logger;
private readonly TimeSpan interval;
public BootstrapInviteCleanupService(
IAuthorityBootstrapInviteStore inviteStore,
IAuthEventSink auditSink,
TimeProvider timeProvider,
ILogger<BootstrapInviteCleanupService> logger)
{
this.inviteStore = inviteStore ?? throw new ArgumentNullException(nameof(inviteStore));
this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
interval = TimeSpan.FromMinutes(5);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var timer = new PeriodicTimer(interval);
try
{
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
{
await SweepExpiredInvitesAsync(stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
// Shutdown requested.
}
finally
{
timer.Dispose();
}
}
internal async Task SweepExpiredInvitesAsync(CancellationToken cancellationToken)
{
var now = timeProvider.GetUtcNow();
var expired = await inviteStore.ExpireAsync(now, cancellationToken).ConfigureAwait(false);
if (expired.Count == 0)
{
return;
}
logger.LogInformation("Expired {Count} bootstrap invite(s).", expired.Count);
foreach (var invite in expired)
{
var record = new AuthEventRecord
{
EventType = "authority.bootstrap.invite.expired",
OccurredAt = now,
CorrelationId = Guid.NewGuid().ToString("N"),
Outcome = AuthEventOutcome.Success,
Reason = "Invite expired before consumption.",
Subject = null,
Client = null,
Scopes = Array.Empty<string>(),
Network = null,
Properties = BuildInviteProperties(invite)
};
await auditSink.WriteAsync(record, cancellationToken).ConfigureAwait(false);
}
}
private static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument invite)
{
var properties = new List<AuthEventProperty>
{
new() { Name = "invite.token", Value = ClassifiedString.Public(invite.Token) },
new() { Name = "invite.type", Value = ClassifiedString.Public(invite.Type) },
new() { Name = "invite.expires_at", Value = ClassifiedString.Public(invite.ExpiresAt.ToString("O", CultureInfo.InvariantCulture)) }
};
if (!string.IsNullOrWhiteSpace(invite.Provider))
{
properties.Add(new AuthEventProperty { Name = "invite.provider", Value = ClassifiedString.Public(invite.Provider) });
}
if (!string.IsNullOrWhiteSpace(invite.Target))
{
properties.Add(new AuthEventProperty { Name = "invite.target", Value = ClassifiedString.Public(invite.Target) });
}
return properties.ToArray();
}
}

View File

@@ -6,6 +6,8 @@ internal sealed record BootstrapUserRequest
{
public string? Provider { get; init; }
public string? InviteToken { get; init; }
[Required]
public string Username { get; init; } = string.Empty;
@@ -27,6 +29,8 @@ internal sealed record BootstrapClientRequest
{
public string? Provider { get; init; }
public string? InviteToken { get; init; }
[Required]
public string ClientId { get; init; } = string.Empty;
@@ -46,3 +50,26 @@ internal sealed record BootstrapClientRequest
public IReadOnlyDictionary<string, string?>? Properties { get; init; }
}
internal sealed record BootstrapInviteRequest
{
public string Type { get; init; } = BootstrapInviteTypes.User;
public string? Token { get; init; }
public string? Provider { get; init; }
public string? Target { get; init; }
public DateTimeOffset? ExpiresAt { get; init; }
public string? IssuedBy { get; init; }
public IReadOnlyDictionary<string, string?>? Metadata { get; init; }
}
internal static class BootstrapInviteTypes
{
public const string User = "user";
public const string Client = "client";
}

View File

@@ -0,0 +1,252 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using OpenIddict.Abstractions;
using OpenIddict.Server;
using StellaOps.Authority.RateLimiting;
using StellaOps.Cryptography.Audit;
namespace StellaOps.Authority.OpenIddict.Handlers;
internal static class ClientCredentialsAuditHelper
{
internal static string EnsureCorrelationId(OpenIddictServerTransaction transaction)
{
ArgumentNullException.ThrowIfNull(transaction);
if (transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditCorrelationProperty, out var value) &&
value is string existing &&
!string.IsNullOrWhiteSpace(existing))
{
return existing;
}
var correlation = Activity.Current?.TraceId.ToString() ??
Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
transaction.Properties[AuthorityOpenIddictConstants.AuditCorrelationProperty] = correlation;
return correlation;
}
internal static AuthEventRecord CreateRecord(
TimeProvider timeProvider,
OpenIddictServerTransaction transaction,
AuthorityRateLimiterMetadata? metadata,
string? clientSecret,
AuthEventOutcome outcome,
string? reason,
string? clientId,
string? providerName,
bool? confidential,
IReadOnlyList<string> requestedScopes,
IReadOnlyList<string> grantedScopes,
string? invalidScope,
IEnumerable<AuthEventProperty>? extraProperties = null,
string? eventType = null)
{
ArgumentNullException.ThrowIfNull(timeProvider);
ArgumentNullException.ThrowIfNull(transaction);
var correlationId = EnsureCorrelationId(transaction);
var client = BuildClient(clientId, providerName);
var network = BuildNetwork(metadata);
var normalizedGranted = NormalizeScopes(grantedScopes);
var properties = BuildProperties(confidential, requestedScopes, invalidScope, extraProperties);
return new AuthEventRecord
{
EventType = string.IsNullOrWhiteSpace(eventType) ? "authority.client_credentials.grant" : eventType,
OccurredAt = timeProvider.GetUtcNow(),
CorrelationId = correlationId,
Outcome = outcome,
Reason = Normalize(reason),
Subject = null,
Client = client,
Scopes = normalizedGranted,
Network = network,
Properties = properties
};
}
internal static AuthEventRecord CreateTamperRecord(
TimeProvider timeProvider,
OpenIddictServerTransaction transaction,
AuthorityRateLimiterMetadata? metadata,
string? clientId,
string? providerName,
bool? confidential,
IEnumerable<string> unexpectedParameters)
{
var properties = new List<AuthEventProperty>
{
new()
{
Name = "request.tampered",
Value = ClassifiedString.Public("true")
}
};
if (confidential.HasValue)
{
properties.Add(new AuthEventProperty
{
Name = "client.confidential",
Value = ClassifiedString.Public(confidential.Value ? "true" : "false")
});
}
if (unexpectedParameters is not null)
{
foreach (var parameter in unexpectedParameters)
{
if (string.IsNullOrWhiteSpace(parameter))
{
continue;
}
properties.Add(new AuthEventProperty
{
Name = "request.unexpected_parameter",
Value = ClassifiedString.Public(parameter)
});
}
}
var reason = unexpectedParameters is null
? "Unexpected parameters supplied to client credentials request."
: $"Unexpected parameters supplied to client credentials request: {string.Join(", ", unexpectedParameters)}.";
return CreateRecord(
timeProvider,
transaction,
metadata,
clientSecret: null,
outcome: AuthEventOutcome.Failure,
reason: reason,
clientId: clientId,
providerName: providerName,
confidential: confidential,
requestedScopes: Array.Empty<string>(),
grantedScopes: Array.Empty<string>(),
invalidScope: null,
extraProperties: properties,
eventType: "authority.token.tamper");
}
private static AuthEventClient? BuildClient(string? clientId, string? providerName)
{
if (string.IsNullOrWhiteSpace(clientId) && string.IsNullOrWhiteSpace(providerName))
{
return null;
}
return new AuthEventClient
{
ClientId = ClassifiedString.Personal(Normalize(clientId)),
Name = ClassifiedString.Empty,
Provider = ClassifiedString.Public(Normalize(providerName))
};
}
private static AuthEventNetwork? BuildNetwork(AuthorityRateLimiterMetadata? metadata)
{
var remote = Normalize(metadata?.RemoteIp);
var forwarded = Normalize(metadata?.ForwardedFor);
var userAgent = Normalize(metadata?.UserAgent);
if (string.IsNullOrWhiteSpace(remote) && string.IsNullOrWhiteSpace(forwarded) && string.IsNullOrWhiteSpace(userAgent))
{
return null;
}
return new AuthEventNetwork
{
RemoteAddress = ClassifiedString.Personal(remote),
ForwardedFor = ClassifiedString.Personal(forwarded),
UserAgent = ClassifiedString.Personal(userAgent)
};
}
private static IReadOnlyList<AuthEventProperty> BuildProperties(
bool? confidential,
IReadOnlyList<string> requestedScopes,
string? invalidScope,
IEnumerable<AuthEventProperty>? extraProperties)
{
var properties = new List<AuthEventProperty>();
if (confidential.HasValue)
{
properties.Add(new AuthEventProperty
{
Name = "client.confidential",
Value = ClassifiedString.Public(confidential.Value ? "true" : "false")
});
}
var normalizedRequested = NormalizeScopes(requestedScopes);
if (normalizedRequested is { Count: > 0 })
{
foreach (var scope in normalizedRequested)
{
if (string.IsNullOrWhiteSpace(scope))
{
continue;
}
properties.Add(new AuthEventProperty
{
Name = "scope.requested",
Value = ClassifiedString.Public(scope)
});
}
}
if (!string.IsNullOrWhiteSpace(invalidScope))
{
properties.Add(new AuthEventProperty
{
Name = "scope.invalid",
Value = ClassifiedString.Public(invalidScope)
});
}
if (extraProperties is not null)
{
foreach (var property in extraProperties)
{
if (property is null || string.IsNullOrWhiteSpace(property.Name))
{
continue;
}
properties.Add(property);
}
}
return properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties;
}
private static IReadOnlyList<string> NormalizeScopes(IReadOnlyList<string>? scopes)
{
if (scopes is null || scopes.Count == 0)
{
return Array.Empty<string>();
}
var normalized = scopes
.Where(static scope => !string.IsNullOrWhiteSpace(scope))
.Select(static scope => scope.Trim())
.Where(static scope => scope.Length > 0)
.Distinct(StringComparer.Ordinal)
.OrderBy(static scope => scope, StringComparer.Ordinal)
.ToArray();
return normalized.Length == 0 ? Array.Empty<string>() : normalized;
}
private static string? Normalize(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
}

View File

@@ -76,6 +76,22 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle
var requestedScopes = requestedScopeInput.IsDefaultOrEmpty ? Array.Empty<string>() : requestedScopeInput.ToArray();
context.Transaction.Properties[AuthorityOpenIddictConstants.AuditRequestedScopesProperty] = requestedScopes;
var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedClientCredentialsParameters(context.Request);
if (unexpectedParameters.Count > 0)
{
var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString();
var tamperRecord = ClientCredentialsAuditHelper.CreateTamperRecord(
timeProvider,
context.Transaction,
metadata,
clientId,
providerHint,
confidential: null,
unexpectedParameters);
await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false);
}
try
{
if (string.IsNullOrWhiteSpace(context.ClientId))

View File

@@ -68,6 +68,23 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler<Op
var requestedScopesInput = context.Request.GetScopes();
var requestedScopes = requestedScopesInput.IsDefaultOrEmpty ? Array.Empty<string>() : requestedScopesInput.ToArray();
var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedPasswordGrantParameters(context.Request);
if (unexpectedParameters.Count > 0)
{
var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString();
var tamperRecord = PasswordGrantAuditHelper.CreateTamperRecord(
timeProvider,
context.Transaction,
metadata,
clientId,
providerHint,
context.Request.Username,
requestedScopes,
unexpectedParameters);
await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false);
}
var selection = AuthorityIdentityProviderSelector.ResolvePasswordProvider(context.Request, registry);
if (!selection.Succeeded)
{
@@ -75,7 +92,6 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler<Op
timeProvider,
context.Transaction,
metadata,
null,
AuthEventOutcome.Failure,
selection.Description,
clientId,
@@ -100,7 +116,6 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler<Op
timeProvider,
context.Transaction,
metadata,
httpContext,
AuthEventOutcome.Failure,
"Both username and password must be provided.",
clientId,
@@ -250,7 +265,6 @@ internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler<Open
timeProvider,
context.Transaction,
metadata,
httpContext,
AuthEventOutcome.Failure,
"Both username and password must be provided.",
clientId,
@@ -395,7 +409,8 @@ internal static class PasswordGrantAuditHelper
IEnumerable<string>? scopes,
TimeSpan? retryAfter,
AuthorityCredentialFailureCode? failureCode,
IEnumerable<AuthEventProperty>? extraProperties)
IEnumerable<AuthEventProperty>? extraProperties,
string? eventType = null)
{
ArgumentNullException.ThrowIfNull(timeProvider);
ArgumentNullException.ThrowIfNull(transaction);
@@ -409,7 +424,7 @@ internal static class PasswordGrantAuditHelper
return new AuthEventRecord
{
EventType = "authority.password.grant",
EventType = string.IsNullOrWhiteSpace(eventType) ? "authority.password.grant" : eventType,
OccurredAt = timeProvider.GetUtcNow(),
CorrelationId = correlationId,
Outcome = outcome,
@@ -581,4 +596,61 @@ internal static class PasswordGrantAuditHelper
private static string? Normalize(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
internal static AuthEventRecord CreateTamperRecord(
TimeProvider timeProvider,
OpenIddictServerTransaction transaction,
AuthorityRateLimiterMetadata? metadata,
string? clientId,
string? providerName,
string? username,
IEnumerable<string>? scopes,
IEnumerable<string> unexpectedParameters)
{
var properties = new List<AuthEventProperty>
{
new()
{
Name = "request.tampered",
Value = ClassifiedString.Public("true")
}
};
if (unexpectedParameters is not null)
{
foreach (var parameter in unexpectedParameters)
{
if (string.IsNullOrWhiteSpace(parameter))
{
continue;
}
properties.Add(new AuthEventProperty
{
Name = "request.unexpected_parameter",
Value = ClassifiedString.Public(parameter)
});
}
}
var reason = unexpectedParameters is null
? "Unexpected parameters supplied to password grant request."
: $"Unexpected parameters supplied to password grant request: {string.Join(", ", unexpectedParameters)}.";
return CreatePasswordGrantRecord(
timeProvider,
transaction,
metadata,
AuthEventOutcome.Failure,
reason,
clientId,
providerName,
user: null,
username,
scopes,
retryAfter: null,
failureCode: null,
extraProperties: properties,
eventType: "authority.token.tamper");
}
}

View File

@@ -111,14 +111,26 @@ internal sealed class HandleRevocationRequestHandler : IOpenIddictServerHandler<
private static byte[] Base64UrlDecode(string value)
{
var padded = value.Length % 4 switch
if (string.IsNullOrWhiteSpace(value))
{
2 => value + "==",
3 => value + "=",
_ => value
};
return Array.Empty<byte>();
}
padded = padded.Replace('-', '+').Replace('_', '/');
var remainder = value.Length % 4;
if (remainder == 2)
{
value += "==";
}
else if (remainder == 3)
{
value += "=";
}
else if (remainder != 0)
{
value += new string('=', 4 - remainder);
}
var padded = value.Replace('-', '+').Replace('_', '/');
return Convert.FromBase64String(padded);
}
}

View File

@@ -119,7 +119,7 @@ internal sealed class PersistTokensHandler : IOpenIddictServerHandler<OpenIddict
private static DateTimeOffset? TryGetExpiration(ClaimsPrincipal principal)
{
var value = principal.GetClaim(OpenIddictConstants.Claims.Exp);
var value = principal.GetClaim("exp");
if (string.IsNullOrWhiteSpace(value))
{
return null;

View File

@@ -1,4 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Security.Claims;
using Microsoft.Extensions.Logging;
using OpenIddict.Abstractions;
@@ -7,8 +10,10 @@ using OpenIddict.Server;
using StellaOps.Auth.Abstractions;
using StellaOps.Authority.OpenIddict;
using StellaOps.Authority.Plugins.Abstractions;
using StellaOps.Authority.RateLimiting;
using StellaOps.Authority.Storage.Mongo.Documents;
using StellaOps.Authority.Storage.Mongo.Stores;
using StellaOps.Cryptography.Audit;
namespace StellaOps.Authority.OpenIddict.Handlers;
@@ -17,6 +22,8 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
private readonly IAuthorityTokenStore tokenStore;
private readonly IAuthorityClientStore clientStore;
private readonly IAuthorityIdentityProviderRegistry registry;
private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor;
private readonly IAuthEventSink auditSink;
private readonly TimeProvider clock;
private readonly ActivitySource activitySource;
private readonly ILogger<ValidateAccessTokenHandler> logger;
@@ -25,6 +32,8 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
IAuthorityTokenStore tokenStore,
IAuthorityClientStore clientStore,
IAuthorityIdentityProviderRegistry registry,
IAuthorityRateLimiterMetadataAccessor metadataAccessor,
IAuthEventSink auditSink,
TimeProvider clock,
ActivitySource activitySource,
ILogger<ValidateAccessTokenHandler> logger)
@@ -32,6 +41,8 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore));
this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore));
this.registry = registry ?? throw new ArgumentNullException(nameof(registry));
this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor));
this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink));
this.clock = clock ?? throw new ArgumentNullException(nameof(clock));
this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -63,9 +74,10 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
? context.TokenId
: context.Principal.GetClaim(OpenIddictConstants.Claims.JwtId);
AuthorityTokenDocument? tokenDocument = null;
if (!string.IsNullOrWhiteSpace(tokenId))
{
var tokenDocument = await tokenStore.FindByTokenIdAsync(tokenId, context.CancellationToken).ConfigureAwait(false);
tokenDocument = await tokenStore.FindByTokenIdAsync(tokenId, context.CancellationToken).ConfigureAwait(false);
if (tokenDocument is not null)
{
if (!string.Equals(tokenDocument.Status, "valid", StringComparison.OrdinalIgnoreCase))
@@ -87,6 +99,11 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
}
}
if (!context.IsRejected && tokenDocument is not null)
{
await TrackTokenUsageAsync(context, tokenDocument, context.Principal).ConfigureAwait(false);
}
var clientId = context.Principal.GetClaim(OpenIddictConstants.Claims.ClientId);
if (!string.IsNullOrWhiteSpace(clientId))
{
@@ -144,4 +161,107 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
identity.GetClaim(OpenIddictConstants.Claims.Subject),
identity.GetClaim(OpenIddictConstants.Claims.ClientId));
}
private async ValueTask TrackTokenUsageAsync(
OpenIddictServerEvents.ValidateTokenContext context,
AuthorityTokenDocument tokenDocument,
ClaimsPrincipal principal)
{
var metadata = metadataAccessor.GetMetadata();
var remoteAddress = metadata?.RemoteIp;
var userAgent = metadata?.UserAgent;
var observedAt = clock.GetUtcNow();
var result = await tokenStore.RecordUsageAsync(tokenDocument.TokenId, remoteAddress, userAgent, observedAt, context.CancellationToken)
.ConfigureAwait(false);
switch (result.Status)
{
case TokenUsageUpdateStatus.MissingMetadata:
logger.LogDebug("Token usage metadata missing for token {TokenId}; replay detection skipped.", tokenDocument.TokenId);
break;
case TokenUsageUpdateStatus.NotFound:
logger.LogWarning("Token usage recording failed: token {TokenId} not found.", tokenDocument.TokenId);
break;
case TokenUsageUpdateStatus.Recorded:
metadataAccessor.SetTag("authority.token_usage", "recorded");
break;
case TokenUsageUpdateStatus.SuspectedReplay:
metadataAccessor.SetTag("authority.token_usage", "suspected_replay");
await EmitReplayAuditAsync(tokenDocument, principal, metadata, result, observedAt, context.CancellationToken).ConfigureAwait(false);
break;
}
}
private async ValueTask EmitReplayAuditAsync(
AuthorityTokenDocument tokenDocument,
ClaimsPrincipal principal,
AuthorityRateLimiterMetadata? metadata,
TokenUsageUpdateResult result,
DateTimeOffset observedAt,
CancellationToken cancellationToken)
{
var clientId = principal.GetClaim(OpenIddictConstants.Claims.ClientId);
var subjectId = principal.GetClaim(OpenIddictConstants.Claims.Subject);
var realm = principal.GetClaim(StellaOpsClaimTypes.IdentityProvider);
var subject = string.IsNullOrWhiteSpace(subjectId) && string.IsNullOrWhiteSpace(realm)
? null
: new AuthEventSubject
{
SubjectId = ClassifiedString.Personal(subjectId),
Realm = ClassifiedString.Public(string.IsNullOrWhiteSpace(realm) ? null : realm)
};
var client = string.IsNullOrWhiteSpace(clientId)
? null
: new AuthEventClient
{
ClientId = ClassifiedString.Personal(clientId)
};
var network = metadata is null && result.RemoteAddress is null && result.UserAgent is null
? null
: new AuthEventNetwork
{
RemoteAddress = ClassifiedString.Personal(result.RemoteAddress ?? metadata?.RemoteIp),
ForwardedFor = ClassifiedString.Personal(metadata?.ForwardedFor),
UserAgent = ClassifiedString.Personal(result.UserAgent ?? metadata?.UserAgent)
};
var previousCount = tokenDocument.Devices?.Count ?? 0;
var properties = new List<AuthEventProperty>
{
new() { Name = "token.id", Value = ClassifiedString.Sensitive(tokenDocument.TokenId) },
new() { Name = "token.type", Value = ClassifiedString.Public(tokenDocument.Type) },
new() { Name = "token.devices.total", Value = ClassifiedString.Public((previousCount + 1).ToString(CultureInfo.InvariantCulture)) }
};
if (!string.IsNullOrWhiteSpace(tokenDocument.ClientId))
{
properties.Add(new AuthEventProperty
{
Name = "token.client_id",
Value = ClassifiedString.Personal(tokenDocument.ClientId)
});
}
logger.LogWarning("Detected suspected token replay for token {TokenId} (client {ClientId}).", tokenDocument.TokenId, clientId ?? "<none>");
var record = new AuthEventRecord
{
EventType = "authority.token.replay.suspected",
OccurredAt = observedAt,
CorrelationId = Activity.Current?.TraceId.ToString() ?? Guid.NewGuid().ToString("N"),
Outcome = AuthEventOutcome.Error,
Reason = "Token observed from a new device fingerprint.",
Subject = subject,
Client = client,
Scopes = Array.Empty<string>(),
Network = network,
Properties = properties
};
await auditSink.WriteAsync(record, cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,112 @@
using System.Collections.Generic;
using System.Linq;
using OpenIddict.Abstractions;
namespace StellaOps.Authority.OpenIddict;
internal static class TokenRequestTamperInspector
{
private static readonly HashSet<string> CommonParameters = new(StringComparer.OrdinalIgnoreCase)
{
OpenIddictConstants.Parameters.GrantType,
OpenIddictConstants.Parameters.Scope,
OpenIddictConstants.Parameters.Resource,
OpenIddictConstants.Parameters.ClientId,
OpenIddictConstants.Parameters.ClientSecret,
OpenIddictConstants.Parameters.ClientAssertion,
OpenIddictConstants.Parameters.ClientAssertionType,
OpenIddictConstants.Parameters.RefreshToken,
OpenIddictConstants.Parameters.DeviceCode,
OpenIddictConstants.Parameters.Code,
OpenIddictConstants.Parameters.CodeVerifier,
OpenIddictConstants.Parameters.CodeChallenge,
OpenIddictConstants.Parameters.CodeChallengeMethod,
OpenIddictConstants.Parameters.RedirectUri,
OpenIddictConstants.Parameters.Assertion,
OpenIddictConstants.Parameters.Nonce,
OpenIddictConstants.Parameters.Prompt,
OpenIddictConstants.Parameters.MaxAge,
OpenIddictConstants.Parameters.UiLocales,
OpenIddictConstants.Parameters.AcrValues,
OpenIddictConstants.Parameters.LoginHint,
OpenIddictConstants.Parameters.Claims,
OpenIddictConstants.Parameters.Token,
OpenIddictConstants.Parameters.TokenTypeHint,
OpenIddictConstants.Parameters.AccessToken,
OpenIddictConstants.Parameters.IdToken
};
private static readonly HashSet<string> PasswordGrantParameters = new(StringComparer.OrdinalIgnoreCase)
{
OpenIddictConstants.Parameters.Username,
OpenIddictConstants.Parameters.Password,
AuthorityOpenIddictConstants.ProviderParameterName
};
private static readonly HashSet<string> ClientCredentialsParameters = new(StringComparer.OrdinalIgnoreCase)
{
AuthorityOpenIddictConstants.ProviderParameterName
};
internal static IReadOnlyList<string> GetUnexpectedPasswordGrantParameters(OpenIddictRequest request)
=> DetectUnexpectedParameters(request, PasswordGrantParameters);
internal static IReadOnlyList<string> GetUnexpectedClientCredentialsParameters(OpenIddictRequest request)
=> DetectUnexpectedParameters(request, ClientCredentialsParameters);
private static IReadOnlyList<string> DetectUnexpectedParameters(
OpenIddictRequest request,
HashSet<string> grantSpecific)
{
if (request is null)
{
return Array.Empty<string>();
}
var unexpected = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var pair in request.GetParameters())
{
var name = pair.Key;
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
if (IsAllowed(name, grantSpecific))
{
continue;
}
unexpected.Add(name);
}
return unexpected.Count == 0
? Array.Empty<string>()
: unexpected
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static bool IsAllowed(string parameterName, HashSet<string> grantSpecific)
{
if (CommonParameters.Contains(parameterName) || grantSpecific.Contains(parameterName))
{
return true;
}
if (parameterName.StartsWith("ext_", StringComparison.OrdinalIgnoreCase) ||
parameterName.StartsWith("x-", StringComparison.OrdinalIgnoreCase) ||
parameterName.StartsWith("custom_", StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (parameterName.Contains(':', StringComparison.Ordinal))
{
return true;
}
return false;
}
}

View File

@@ -24,6 +24,7 @@ using StellaOps.Authority.Plugins;
using StellaOps.Authority.Bootstrap;
using StellaOps.Authority.Storage.Mongo.Extensions;
using StellaOps.Authority.Storage.Mongo.Initialization;
using StellaOps.Authority.Storage.Mongo.Stores;
using StellaOps.Authority.RateLimiting;
using StellaOps.Configuration;
using StellaOps.Plugin.DependencyInjection;
@@ -35,6 +36,7 @@ using StellaOps.Cryptography.DependencyInjection;
using StellaOps.Authority.Revocation;
using StellaOps.Authority.Signing;
using StellaOps.Cryptography;
using StellaOps.Authority.Storage.Mongo.Documents;
var builder = WebApplication.CreateBuilder(args);
@@ -124,6 +126,7 @@ builder.Services.AddSingleton<RevocationBundleBuilder>();
builder.Services.AddSingleton<RevocationBundleSigner>();
builder.Services.AddSingleton<AuthorityRevocationExportService>();
builder.Services.AddSingleton<AuthorityJwksService>();
builder.Services.AddHostedService<BootstrapInviteCleanupService>();
var pluginRegistrationSummary = AuthorityPluginLoader.RegisterPlugins(
builder.Services,
@@ -281,38 +284,98 @@ if (authorityOptions.Bootstrap.Enabled)
HttpContext httpContext,
BootstrapUserRequest request,
IAuthorityIdentityProviderRegistry registry,
IAuthorityBootstrapInviteStore inviteStore,
IAuthEventSink auditSink,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (request is null)
{
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>()).ConfigureAwait(false);
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>(), null).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." });
}
var now = timeProvider.GetUtcNow();
var inviteToken = string.IsNullOrWhiteSpace(request.InviteToken) ? null : request.InviteToken.Trim();
AuthorityBootstrapInviteDocument? invite = null;
var inviteReserved = false;
async Task ReleaseInviteAsync(string reason)
{
if (inviteToken is null)
{
return;
}
if (inviteReserved)
{
await inviteStore.ReleaseAsync(inviteToken, cancellationToken).ConfigureAwait(false);
}
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, reason, invite, inviteToken).ConfigureAwait(false);
}
if (inviteToken is not null)
{
var reservation = await inviteStore.TryReserveAsync(inviteToken, BootstrapInviteTypes.User, now, request.Username, cancellationToken).ConfigureAwait(false);
switch (reservation.Status)
{
case BootstrapInviteReservationStatus.Reserved:
inviteReserved = true;
invite = reservation.Invite;
break;
case BootstrapInviteReservationStatus.Expired:
await WriteInviteAuditAsync("authority.bootstrap.invite.expired", AuthEventOutcome.Failure, "Invite expired before use.", reservation.Invite, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invite_expired", message = "Invite has expired." });
case BootstrapInviteReservationStatus.AlreadyUsed:
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token already consumed.", reservation.Invite, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invite_used", message = "Invite token has already been used." });
default:
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token not found.", reservation.Invite, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_invite", message = "Invite token is invalid." });
}
}
var providerName = string.IsNullOrWhiteSpace(request.Provider)
? authorityOptions.Bootstrap.DefaultIdentityProvider
? invite?.Provider ?? authorityOptions.Bootstrap.DefaultIdentityProvider
: request.Provider;
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Provider) &&
!string.Equals(invite.Provider, providerName, StringComparison.OrdinalIgnoreCase))
{
await ReleaseInviteAsync("Invite provider does not match requested provider.");
return Results.BadRequest(new { error = "invite_provider_mismatch", message = "Invite is limited to a different identity provider." });
}
if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider))
{
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", null, request.Username, providerName, request.Roles ?? Array.Empty<string>()).ConfigureAwait(false);
await ReleaseInviteAsync("Specified identity provider was not found.");
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", null, request.Username, providerName, request.Roles ?? Array.Empty<string>(), inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." });
}
if (!provider.Capabilities.SupportsPassword)
{
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support password provisioning.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>()).ConfigureAwait(false);
await ReleaseInviteAsync("Selected provider does not support password provisioning.");
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support password provisioning.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>(), inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support password provisioning." });
}
if (string.IsNullOrWhiteSpace(request.Username) || string.IsNullOrEmpty(request.Password))
{
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Username and password are required.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>()).ConfigureAwait(false);
await ReleaseInviteAsync("Username and password are required.");
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Username and password are required.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>(), inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = "Username and password are required." });
}
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Target) &&
!string.Equals(invite.Target, request.Username, StringComparison.OrdinalIgnoreCase))
{
await ReleaseInviteAsync("Invite target does not match requested username.");
return Results.BadRequest(new { error = "invite_target_mismatch", message = "Invite target does not match username." });
}
var roles = request.Roles is null ? Array.Empty<string>() : request.Roles.ToArray();
var attributes = request.Attributes is null
? new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
@@ -327,24 +390,47 @@ if (authorityOptions.Bootstrap.Enabled)
roles,
attributes);
var result = await provider.Credentials.UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false);
if (!result.Succeeded || result.Value is null)
try
{
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, result.Message ?? "User provisioning failed.", null, request.Username, provider.Name, roles).ConfigureAwait(false);
return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "User provisioning failed." });
var result = await provider.Credentials.UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false);
if (!result.Succeeded || result.Value is null)
{
await ReleaseInviteAsync(result.Message ?? "User provisioning failed.");
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, result.Message ?? "User provisioning failed.", null, request.Username, provider.Name, roles, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "User provisioning failed." });
}
if (inviteReserved && inviteToken is not null)
{
var consumed = await inviteStore.MarkConsumedAsync(inviteToken, result.Value.SubjectId ?? result.Value.Username, now, cancellationToken).ConfigureAwait(false);
if (consumed)
{
await WriteInviteAuditAsync("authority.bootstrap.invite.consumed", AuthEventOutcome.Success, null, invite, inviteToken).ConfigureAwait(false);
}
}
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Success, null, result.Value.SubjectId, result.Value.Username, provider.Name, roles, inviteToken).ConfigureAwait(false);
return Results.Ok(new
{
provider = provider.Name,
subjectId = result.Value.SubjectId,
username = result.Value.Username
});
}
catch
{
if (inviteReserved && inviteToken is not null)
{
await inviteStore.ReleaseAsync(inviteToken, cancellationToken).ConfigureAwait(false);
await WriteInviteAuditAsync("authority.bootstrap.invite.released", AuthEventOutcome.Error, "Invite released due to provisioning failure.", invite, inviteToken).ConfigureAwait(false);
}
throw;
}
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Success, null, result.Value.SubjectId, result.Value.Username, provider.Name, roles).ConfigureAwait(false);
return Results.Ok(new
{
provider = provider.Name,
subjectId = result.Value.SubjectId,
username = result.Value.Username
});
async Task WriteBootstrapUserAuditAsync(AuthEventOutcome outcome, string? reason, string? subjectId, string? usernameValue, string? providerValue, IReadOnlyCollection<string> rolesValue)
async Task WriteBootstrapUserAuditAsync(AuthEventOutcome outcome, string? reason, string? subjectId, string? usernameValue, string? providerValue, IReadOnlyCollection<string> rolesValue, string? inviteValue)
{
var correlationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
AuthEventNetwork? network = null;
@@ -369,16 +455,24 @@ if (authorityOptions.Bootstrap.Enabled)
Realm = ClassifiedString.Public(providerValue)
};
var properties = string.IsNullOrWhiteSpace(providerValue)
? Array.Empty<AuthEventProperty>()
: new[]
var properties = new List<AuthEventProperty>();
if (!string.IsNullOrWhiteSpace(providerValue))
{
properties.Add(new AuthEventProperty
{
new AuthEventProperty
{
Name = "bootstrap.provider",
Value = ClassifiedString.Public(providerValue)
}
};
Name = "bootstrap.provider",
Value = ClassifiedString.Public(providerValue)
});
}
if (!string.IsNullOrWhiteSpace(inviteValue))
{
properties.Add(new AuthEventProperty
{
Name = "bootstrap.invite_token",
Value = ClassifiedString.Public(inviteValue)
});
}
var scopes = rolesValue is { Count: > 0 }
? rolesValue.ToArray()
@@ -395,65 +489,199 @@ if (authorityOptions.Bootstrap.Enabled)
Client = null,
Scopes = scopes,
Network = network,
Properties = properties
Properties = properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties
};
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
}
async Task WriteInviteAuditAsync(string eventType, AuthEventOutcome outcome, string? reason, AuthorityBootstrapInviteDocument? document, string? tokenValue)
{
var record = new AuthEventRecord
{
EventType = eventType,
OccurredAt = timeProvider.GetUtcNow(),
CorrelationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture),
Outcome = outcome,
Reason = reason,
Subject = null,
Client = null,
Scopes = Array.Empty<string>(),
Network = null,
Properties = BuildInviteProperties(document, tokenValue)
};
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
}
static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument? document, string? token)
{
var properties = new List<AuthEventProperty>();
if (!string.IsNullOrWhiteSpace(token))
{
properties.Add(new AuthEventProperty
{
Name = "invite.token",
Value = ClassifiedString.Public(token)
});
}
if (document is not null)
{
if (!string.IsNullOrWhiteSpace(document.Type))
{
properties.Add(new AuthEventProperty
{
Name = "invite.type",
Value = ClassifiedString.Public(document.Type)
});
}
if (!string.IsNullOrWhiteSpace(document.Provider))
{
properties.Add(new AuthEventProperty
{
Name = "invite.provider",
Value = ClassifiedString.Public(document.Provider)
});
}
if (!string.IsNullOrWhiteSpace(document.Target))
{
properties.Add(new AuthEventProperty
{
Name = "invite.target",
Value = ClassifiedString.Public(document.Target)
});
}
properties.Add(new AuthEventProperty
{
Name = "invite.expires_at",
Value = ClassifiedString.Public(document.ExpiresAt.ToString("O", CultureInfo.InvariantCulture))
});
}
return properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties.ToArray();
}
});
bootstrapGroup.MapPost("/clients", async (
HttpContext httpContext,
BootstrapClientRequest request,
IAuthorityIdentityProviderRegistry registry,
IAuthorityBootstrapInviteStore inviteStore,
IAuthEventSink auditSink,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (request is null)
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>(), null).ConfigureAwait(false);
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>(), null, null).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." });
}
var now = timeProvider.GetUtcNow();
var inviteToken = string.IsNullOrWhiteSpace(request.InviteToken) ? null : request.InviteToken.Trim();
AuthorityBootstrapInviteDocument? invite = null;
var inviteReserved = false;
async Task ReleaseInviteAsync(string reason)
{
if (inviteToken is null)
{
return;
}
if (inviteReserved)
{
await inviteStore.ReleaseAsync(inviteToken, cancellationToken).ConfigureAwait(false);
}
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, reason, invite, inviteToken).ConfigureAwait(false);
}
if (inviteToken is not null)
{
var reservation = await inviteStore.TryReserveAsync(inviteToken, BootstrapInviteTypes.Client, now, request.ClientId, cancellationToken).ConfigureAwait(false);
switch (reservation.Status)
{
case BootstrapInviteReservationStatus.Reserved:
inviteReserved = true;
invite = reservation.Invite;
break;
case BootstrapInviteReservationStatus.Expired:
await WriteInviteAuditAsync("authority.bootstrap.invite.expired", AuthEventOutcome.Failure, "Invite expired before use.", reservation.Invite, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invite_expired", message = "Invite has expired." });
case BootstrapInviteReservationStatus.AlreadyUsed:
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token already consumed.", reservation.Invite, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invite_used", message = "Invite token has already been used." });
default:
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token is invalid.", reservation.Invite, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_invite", message = "Invite token is invalid." });
}
}
var providerName = string.IsNullOrWhiteSpace(request.Provider)
? authorityOptions.Bootstrap.DefaultIdentityProvider
? invite?.Provider ?? authorityOptions.Bootstrap.DefaultIdentityProvider
: request.Provider;
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Provider) &&
!string.Equals(invite.Provider, providerName, StringComparison.OrdinalIgnoreCase))
{
await ReleaseInviteAsync("Invite provider does not match requested provider.");
return Results.BadRequest(new { error = "invite_provider_mismatch", message = "Invite is limited to a different identity provider." });
}
if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider))
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", request.ClientId, null, providerName, request.AllowedScopes ?? Array.Empty<string>(), request?.Confidential).ConfigureAwait(false);
await ReleaseInviteAsync("Specified identity provider was not found.");
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", request.ClientId, null, providerName, request.AllowedScopes ?? Array.Empty<string>(), request?.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." });
}
if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null)
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
await ReleaseInviteAsync("Selected provider does not support client provisioning.");
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support client provisioning." });
}
if (string.IsNullOrWhiteSpace(request.ClientId))
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "ClientId is required.", null, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
await ReleaseInviteAsync("ClientId is required.");
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "ClientId is required.", null, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = "ClientId is required." });
}
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Target) &&
!string.Equals(invite.Target, request.ClientId, StringComparison.OrdinalIgnoreCase))
{
await ReleaseInviteAsync("Invite target does not match requested client id.");
return Results.BadRequest(new { error = "invite_target_mismatch", message = "Invite target does not match client id." });
}
if (request.Confidential && string.IsNullOrWhiteSpace(request.ClientSecret))
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Confidential clients require a client secret.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
await ReleaseInviteAsync("Confidential clients require a client secret.");
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Confidential clients require a client secret.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = "Confidential clients require a client secret." });
}
if (!TryParseUris(request.RedirectUris, out var redirectUris, out var redirectError))
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, redirectError, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = redirectError });
var errorMessage = redirectError ?? "Redirect URI validation failed.";
await ReleaseInviteAsync(errorMessage);
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = errorMessage });
}
if (!TryParseUris(request.PostLogoutRedirectUris, out var postLogoutUris, out var postLogoutError))
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, postLogoutError, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = postLogoutError });
var errorMessage = postLogoutError ?? "Post-logout redirect URI validation failed.";
await ReleaseInviteAsync(errorMessage);
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = "invalid_request", message = errorMessage });
}
var properties = request.Properties is null
@@ -475,11 +703,21 @@ if (authorityOptions.Bootstrap.Enabled)
if (!result.Succeeded || result.Value is null)
{
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, result.Message ?? "Client provisioning failed.", request.ClientId, result.Value?.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
await ReleaseInviteAsync(result.Message ?? "Client provisioning failed.");
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, result.Message ?? "Client provisioning failed.", request.ClientId, result.Value?.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "Client provisioning failed." });
}
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Success, null, request.ClientId, result.Value.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
if (inviteReserved && inviteToken is not null)
{
var consumed = await inviteStore.MarkConsumedAsync(inviteToken, result.Value.ClientId, now, cancellationToken).ConfigureAwait(false);
if (consumed)
{
await WriteInviteAuditAsync("authority.bootstrap.invite.consumed", AuthEventOutcome.Success, null, invite, inviteToken).ConfigureAwait(false);
}
}
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Success, null, request.ClientId, result.Value.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
return Results.Ok(new
{
@@ -488,7 +726,7 @@ if (authorityOptions.Bootstrap.Enabled)
confidential = result.Value.Confidential
});
async Task WriteBootstrapClientAuditAsync(AuthEventOutcome outcome, string? reason, string? requestedClientId, string? assignedClientId, string? providerValue, IReadOnlyCollection<string> scopes, bool? confidentialFlag)
async Task WriteBootstrapClientAuditAsync(AuthEventOutcome outcome, string? reason, string? requestedClientId, string? assignedClientId, string? providerValue, IReadOnlyCollection<string> scopes, bool? confidentialFlag, string? inviteValue)
{
var correlationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
AuthEventNetwork? network = null;
@@ -533,6 +771,15 @@ if (authorityOptions.Bootstrap.Enabled)
});
}
if (!string.IsNullOrWhiteSpace(inviteValue))
{
properties.Add(new AuthEventProperty
{
Name = "bootstrap.invite_token",
Value = ClassifiedString.Public(inviteValue)
});
}
var record = new AuthEventRecord
{
EventType = "authority.bootstrap.client",
@@ -549,6 +796,175 @@ if (authorityOptions.Bootstrap.Enabled)
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
}
async Task WriteInviteAuditAsync(string eventType, AuthEventOutcome outcome, string? reason, AuthorityBootstrapInviteDocument? document, string? tokenValue)
{
var record = new AuthEventRecord
{
EventType = eventType,
OccurredAt = timeProvider.GetUtcNow(),
CorrelationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture),
Outcome = outcome,
Reason = reason,
Subject = null,
Client = null,
Scopes = Array.Empty<string>(),
Network = null,
Properties = BuildInviteProperties(document, tokenValue)
};
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
}
static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument? document, string? token)
{
var properties = new List<AuthEventProperty>();
if (!string.IsNullOrWhiteSpace(token))
{
properties.Add(new AuthEventProperty
{
Name = "invite.token",
Value = ClassifiedString.Public(token)
});
}
if (document is not null)
{
if (!string.IsNullOrWhiteSpace(document.Type))
{
properties.Add(new AuthEventProperty
{
Name = "invite.type",
Value = ClassifiedString.Public(document.Type)
});
}
if (!string.IsNullOrWhiteSpace(document.Provider))
{
properties.Add(new AuthEventProperty
{
Name = "invite.provider",
Value = ClassifiedString.Public(document.Provider)
});
}
if (!string.IsNullOrWhiteSpace(document.Target))
{
properties.Add(new AuthEventProperty
{
Name = "invite.target",
Value = ClassifiedString.Public(document.Target)
});
}
properties.Add(new AuthEventProperty
{
Name = "invite.expires_at",
Value = ClassifiedString.Public(document.ExpiresAt.ToString("O", CultureInfo.InvariantCulture))
});
}
return properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties.ToArray();
}
});
bootstrapGroup.MapPost("/invites", async (
HttpContext httpContext,
BootstrapInviteRequest request,
IAuthorityBootstrapInviteStore inviteStore,
IAuthEventSink auditSink,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (request is null)
{
return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." });
}
if (string.IsNullOrWhiteSpace(request.Type) ||
( !string.Equals(request.Type, BootstrapInviteTypes.User, StringComparison.OrdinalIgnoreCase) &&
!string.Equals(request.Type, BootstrapInviteTypes.Client, StringComparison.OrdinalIgnoreCase)))
{
return Results.BadRequest(new { error = "invalid_request", message = "Invite type must be 'user' or 'client'." });
}
var now = timeProvider.GetUtcNow();
var expiresAt = request.ExpiresAt ?? now.AddDays(2);
if (expiresAt <= now)
{
return Results.BadRequest(new { error = "invalid_request", message = "ExpiresAt must be in the future." });
}
var token = string.IsNullOrWhiteSpace(request.Token) ? Guid.NewGuid().ToString("N") : request.Token.Trim();
var document = new AuthorityBootstrapInviteDocument
{
Token = token,
Type = request.Type.ToLowerInvariant(),
Provider = string.IsNullOrWhiteSpace(request.Provider) ? null : request.Provider.Trim(),
Target = string.IsNullOrWhiteSpace(request.Target) ? null : request.Target.Trim(),
IssuedAt = now,
IssuedBy = string.IsNullOrWhiteSpace(request.IssuedBy) ? httpContext.User?.Identity?.Name : request.IssuedBy,
ExpiresAt = expiresAt,
Metadata = request.Metadata is null ? null : new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase)
};
await inviteStore.CreateAsync(document, cancellationToken).ConfigureAwait(false);
await WriteInviteAuditAsync("authority.bootstrap.invite.created", AuthEventOutcome.Success, null, document).ConfigureAwait(false);
return Results.Ok(new
{
document.Token,
document.Type,
document.Provider,
document.Target,
document.ExpiresAt
});
async Task WriteInviteAuditAsync(string eventType, AuthEventOutcome outcome, string? reason, AuthorityBootstrapInviteDocument invite)
{
var record = new AuthEventRecord
{
EventType = eventType,
OccurredAt = timeProvider.GetUtcNow(),
CorrelationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture),
Outcome = outcome,
Reason = reason,
Subject = null,
Client = null,
Scopes = Array.Empty<string>(),
Network = null,
Properties = BuildInviteProperties(invite)
};
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
}
static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument invite)
{
var properties = new List<AuthEventProperty>
{
new() { Name = "invite.token", Value = ClassifiedString.Public(invite.Token) },
new() { Name = "invite.type", Value = ClassifiedString.Public(invite.Type) },
new() { Name = "invite.expires_at", Value = ClassifiedString.Public(invite.ExpiresAt.ToString("O", CultureInfo.InvariantCulture)) }
};
if (!string.IsNullOrWhiteSpace(invite.Provider))
{
properties.Add(new AuthEventProperty { Name = "invite.provider", Value = ClassifiedString.Public(invite.Provider) });
}
if (!string.IsNullOrWhiteSpace(invite.Target))
{
properties.Add(new AuthEventProperty { Name = "invite.target", Value = ClassifiedString.Public(invite.Target) });
}
if (!string.IsNullOrWhiteSpace(invite.IssuedBy))
{
properties.Add(new AuthEventProperty { Name = "invite.issued_by", Value = ClassifiedString.Public(invite.IssuedBy) });
}
return properties.ToArray();
}
});
bootstrapGroup.MapGet("/revocations/export", async (
@@ -573,6 +989,7 @@ if (authorityOptions.Bootstrap.Enabled)
{
Algorithm = package.Signature.Algorithm,
KeyId = package.Signature.KeyId,
Provider = package.Signature.Provider,
Value = package.Signature.Value
},
Digest = new RevocationExportDigest

View File

@@ -41,6 +41,11 @@ internal sealed class AuthorityRateLimiterMetadata
/// </summary>
public IReadOnlyDictionary<string, string> Tags => tags;
/// <summary>
/// User agent string associated with the request, if captured.
/// </summary>
public string? UserAgent { get; set; }
/// <summary>
/// Adds or updates an arbitrary metadata tag for downstream consumers.
/// </summary>

View File

@@ -61,6 +61,9 @@ internal sealed class AuthorityRateLimiterMetadataMiddleware
metadata.ClientId = ResolveAuthorizeClientId(context.Request.Query);
}
var userAgent = NormalizeUserAgent(context.Request.Headers.UserAgent.ToString());
metadata.UserAgent = userAgent;
if (!string.IsNullOrWhiteSpace(metadata.ClientId))
{
metadata.SetTag("authority.client_id", metadata.ClientId);
@@ -74,6 +77,10 @@ internal sealed class AuthorityRateLimiterMetadataMiddleware
metadata.SetTag("authority.endpoint", metadata.Endpoint ?? string.Empty);
metadata.SetTag("authority.remote_ip", metadata.RemoteIp ?? "unknown");
metadata.SetTag("authority.captured_at", clock.GetUtcNow().ToString("O", CultureInfo.InvariantCulture));
if (!string.IsNullOrWhiteSpace(userAgent))
{
metadata.SetTag("authority.user_agent", userAgent);
}
await next(context).ConfigureAwait(false);
}
@@ -145,6 +152,17 @@ internal sealed class AuthorityRateLimiterMetadataMiddleware
return null;
}
private static string? NormalizeUserAgent(string? userAgent)
{
if (string.IsNullOrWhiteSpace(userAgent))
{
return null;
}
var trimmed = userAgent.Trim();
return trimmed.Length == 0 ? null : trimmed;
}
private async Task<string?> ResolveTokenClientIdAsync(HttpContext context)
{
var request = context.Request;

View File

@@ -1,3 +1,3 @@
namespace StellaOps.Authority.Revocation;
internal sealed record RevocationBundleSignature(string Algorithm, string KeyId, string Value);
internal sealed record RevocationBundleSignature(string Algorithm, string KeyId, string Provider, string Value);

View File

@@ -51,12 +51,18 @@ internal sealed class RevocationBundleSigner
: signing.Algorithm.Trim();
var keyReference = new CryptoKeyReference(signing.ActiveKeyId, signing.Provider);
var signer = providerRegistry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, signing.Provider);
var resolved = providerRegistry.ResolveSigner(
CryptoCapability.Signing,
algorithm,
keyReference,
signing.Provider);
var signer = resolved.Signer;
var header = new Dictionary<string, object>
{
["alg"] = algorithm,
["kid"] = signing.ActiveKeyId,
["provider"] = resolved.ProviderName,
["typ"] = "application/vnd.stellaops.revocation-bundle+jws",
["b64"] = false,
["crit"] = new[] { "b64" }
@@ -77,7 +83,11 @@ internal sealed class RevocationBundleSigner
var signingInput = new ReadOnlyMemory<byte>(buffer, 0, signingInputLength);
var signatureBytes = await signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false);
var encodedSignature = Base64UrlEncode(signatureBytes);
return new RevocationBundleSignature(algorithm, signing.ActiveKeyId, string.Concat(protectedHeader, "..", encodedSignature));
return new RevocationBundleSignature(
algorithm,
signing.ActiveKeyId,
resolved.ProviderName,
string.Concat(protectedHeader, "..", encodedSignature));
}
finally
{

View File

@@ -56,6 +56,9 @@ internal sealed class RevocationExportSignature
[JsonPropertyName("keyId")]
public required string KeyId { get; init; }
[JsonPropertyName("provider")]
public required string Provider { get; init; }
[JsonPropertyName("value")]
public required string Value { get; init; }
}

View File

@@ -236,10 +236,11 @@ internal sealed class AuthoritySigningKeyManager
["status"] = AuthoritySigningKeyStatus.Retired
};
var privateParameters = previous.Key.PrivateParameters;
var retiredKey = new CryptoSigningKey(
previous.Key.Reference,
previous.Key.AlgorithmId,
in previous.Key.PrivateParameters,
in privateParameters,
previous.Key.CreatedAt,
previous.Key.ExpiresAt,
metadata);

View File

@@ -6,11 +6,18 @@
| CORE9.REVOCATION | DONE (2025-10-12) | Authority Core, Security Guild | CORE5 | Implement revocation list persistence + export hooks (API + CLI). | ✅ Revoked tokens denied; ✅ Export endpoint/CLI returns manifest; ✅ Tests cover offline bundle flow. |
| CORE10.JWKS | DONE (2025-10-12) | Authority Core, DevOps | CORE9.REVOCATION | Provide JWKS rotation with pluggable key loader + documentation. | ✅ Signing/encryption keys rotate without downtime; ✅ JWKS endpoint updates; ✅ Docs describe rotation SOP. |
| CORE8.RL | DONE (2025-10-12) | Authority Core | CORE8 | Deliver ASP.NET rate limiter plumbing (request metadata, dependency injection hooks) needed by Security Guild. | ✅ `/token` & `/authorize` pipelines expose limiter hooks; ✅ Tests cover throttle behaviour baseline. |
| SEC2.HOST | TODO | Security Guild, Authority Core | SEC2.A (audit contract) | Hook audit logger into OpenIddict handlers and bootstrap endpoints. | ✅ Audit events populated with correlationId, IP, client_id; ✅ Mongo login attempts persisted; ✅ Tests verify on success/failure/lockout. |
| SEC2.HOST | DONE (2025-10-12) | Security Guild, Authority Core | SEC2.A (audit contract) | Hook audit logger into OpenIddict handlers and bootstrap endpoints. | ✅ Audit events populated with correlationId, IP, client_id; ✅ Mongo login attempts persisted; ✅ Tests verify on success/failure/lockout. |
| SEC3.HOST | DONE (2025-10-11) | Security Guild | CORE8.RL, SEC3.A (rate policy) | Apply rate limiter policies (`AddRateLimiter`) to `/token` and `/internal/*` endpoints with configuration binding. | ✅ Policies configurable via `StellaOpsAuthorityOptions.Security.RateLimiting`; ✅ Integration tests hit 429 after limit; ✅ Docs updated. |
| SEC4.HOST | DONE (2025-10-12) | Security Guild, DevOps | SEC4.A (revocation schema) | Implement CLI/HTTP surface to export revocation bundle + detached JWS using `StellaOps.Cryptography`. | ✅ `stellaops auth revoke export` CLI/endpoint returns JSON + `.jws`; ✅ Verification script passes; ✅ Operator docs updated. |
| SEC4.KEY | DONE (2025-10-12) | Security Guild, DevOps | SEC4.HOST | Integrate signing keys with provider registry (initial ES256). | ✅ Keys loaded via `ICryptoProvider` signer; ✅ Rotation SOP documented. |
| SEC5.HOST | TODO | Security Guild | SEC5.A (threat model) | Feed Authority-specific mitigations (rate limiting, audit, revocation) into threat model + backlog. | ✅ Threat model updated; ✅ Backlog issues reference mitigations; ✅ Review sign-off captured. |
| SEC5.HOST | DONE (2025-10-14) | Security Guild | SEC5.A (threat model) | Feed Authority-specific mitigations (rate limiting, audit, revocation) into threat model + backlog. | ✅ Threat model updated; ✅ Backlog issues reference mitigations; ✅ Review sign-off captured. |
| SEC5.HOST-INVITES | DONE (2025-10-14) | Security Guild, Authority Core | SEC5.D | Implement bootstrap invite persistence, APIs, and background cleanup with audit coverage. | ✅ Invite store + endpoints complete; ✅ Cleanup service expires unused invites; ✅ Audit events for create/consume/expire; ✅ Build/tests green. |
> Remark (2025-10-14): Background sweep emits invite expiry audits; integration test added.
| SEC5.HOST-REPLAY | DONE (2025-10-14) | Security Guild, Zastava | SEC5.E | Persist token usage metadata and surface suspected replay heuristics. | ✅ Validation handlers record device metadata; ✅ Suspected replay flagged via audit/logs; ✅ Tests cover regression cases. |
> Remark (2025-10-14): Token validation handler logs suspected replay audits with device metadata; coverage via unit/integration tests.
| SEC3.BUILD | DONE (2025-10-11) | Authority Core, Security Guild | SEC3.HOST, FEEDMERGE-COORD-02-900 | Track normalized-range dependency fallout and restore full test matrix once Feedser range primitives land. | ✅ Feedser normalized range libraries merged; ✅ Authority + Configuration test suites (`dotnet test src/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) pass without Feedser compile failures; ✅ Status recorded here/Sprints (authority-core broadcast not available). |
| AUTHCORE-BUILD-OPENIDDICT | DONE (2025-10-14) | Authority Core | SEC2.HOST | Adapt host/audit handlers for OpenIddict 6.4 API surface (no `OpenIddictServerTransaction`) and restore Authority solution build. | ✅ Build `dotnet build src/StellaOps.Authority.sln` succeeds; ✅ Audit correlation + tamper logging verified under new abstractions; ✅ Tests updated. |
| AUTHCORE-STORAGE-DEVICE-TOKENS | DONE (2025-10-14) | Authority Core, Storage Guild | AUTHCORE-BUILD-OPENIDDICT | Reintroduce `AuthorityTokenDeviceDocument` + projections removed during refactor so storage layer compiles. | ✅ Document type restored with mappings/migrations; ✅ Storage tests cover device artifacts; ✅ Authority solution build green. |
| AUTHCORE-BOOTSTRAP-INVITES | DONE (2025-10-14) | Authority Core, DevOps | AUTHCORE-STORAGE-DEVICE-TOKENS | Wire bootstrap invite cleanup service against restored document schema and re-enable lifecycle tests. | ✅ `BootstrapInviteCleanupService` passes integration tests; ✅ Operator guide updated if behavior changes; ✅ Build/test matrices green. |
> Update status columns (TODO / DOING / DONE / BLOCKED) together with code changes. Always run `dotnet test src/StellaOps.Authority.sln` when touching host logic.

View File

@@ -1,8 +1,10 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
@@ -16,6 +18,7 @@ using StellaOps.Cli.Services;
using StellaOps.Cli.Services.Models;
using StellaOps.Cli.Telemetry;
using StellaOps.Cli.Tests.Testing;
using StellaOps.Cryptography;
namespace StellaOps.Cli.Tests.Commands;
@@ -208,6 +211,34 @@ public sealed class CommandHandlersTests
}
}
[Theory]
[InlineData(null)]
[InlineData("default")]
[InlineData("libsodium")]
public async Task HandleAuthRevokeVerifyAsync_VerifiesBundlesUsingProviderRegistry(string? providerHint)
{
var original = Environment.ExitCode;
using var tempDir = new TempDirectory();
try
{
var artifacts = await WriteRevocationArtifactsAsync(tempDir, providerHint);
await CommandHandlers.HandleAuthRevokeVerifyAsync(
artifacts.BundlePath,
artifacts.SignaturePath,
artifacts.KeyPath,
verbose: true,
cancellationToken: CancellationToken.None);
Assert.Equal(0, Environment.ExitCode);
}
finally
{
Environment.ExitCode = original;
}
}
[Fact]
public async Task HandleAuthStatusAsync_ReportsCachedToken()
{
@@ -360,6 +391,79 @@ public sealed class CommandHandlersTests
}
}
private static async Task<RevocationArtifactPaths> WriteRevocationArtifactsAsync(TempDirectory temp, string? providerHint)
{
var (bundleBytes, signature, keyPem) = await BuildRevocationArtifactsAsync(providerHint);
var bundlePath = Path.Combine(temp.Path, "revocation-bundle.json");
var signaturePath = Path.Combine(temp.Path, "revocation-bundle.json.jws");
var keyPath = Path.Combine(temp.Path, "revocation-key.pem");
await File.WriteAllBytesAsync(bundlePath, bundleBytes);
await File.WriteAllTextAsync(signaturePath, signature);
await File.WriteAllTextAsync(keyPath, keyPem);
return new RevocationArtifactPaths(bundlePath, signaturePath, keyPath);
}
private static async Task<(byte[] Bundle, string Signature, string KeyPem)> BuildRevocationArtifactsAsync(string? providerHint)
{
var bundleBytes = Encoding.UTF8.GetBytes("{\"revocations\":[]}");
using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
var signingKey = new CryptoSigningKey(
new CryptoKeyReference("revocation-test"),
SignatureAlgorithms.Es256,
privateParameters: in parameters,
createdAt: DateTimeOffset.UtcNow);
var provider = new DefaultCryptoProvider();
provider.UpsertSigningKey(signingKey);
var signer = provider.GetSigner(SignatureAlgorithms.Es256, signingKey.Reference);
var header = new Dictionary<string, object>
{
["alg"] = SignatureAlgorithms.Es256,
["kid"] = signingKey.Reference.KeyId,
["typ"] = "application/vnd.stellaops.revocation-bundle+jws",
["b64"] = false,
["crit"] = new[] { "b64" }
};
if (!string.IsNullOrWhiteSpace(providerHint))
{
header["provider"] = providerHint;
}
var serializerOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = null,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
var headerJson = JsonSerializer.Serialize(header, serializerOptions);
var encodedHeader = Base64UrlEncoder.Encode(Encoding.UTF8.GetBytes(headerJson));
var signingInput = new byte[encodedHeader.Length + 1 + bundleBytes.Length];
var headerBytes = Encoding.ASCII.GetBytes(encodedHeader);
Buffer.BlockCopy(headerBytes, 0, signingInput, 0, headerBytes.Length);
signingInput[headerBytes.Length] = (byte)'.';
Buffer.BlockCopy(bundleBytes, 0, signingInput, headerBytes.Length + 1, bundleBytes.Length);
var signatureBytes = await signer.SignAsync(signingInput);
var encodedSignature = Base64UrlEncoder.Encode(signatureBytes);
var jws = string.Concat(encodedHeader, "..", encodedSignature);
var publicKeyBytes = ecdsa.ExportSubjectPublicKeyInfo();
var keyPem = new string(PemEncoding.Write("PUBLIC KEY", publicKeyBytes));
return (bundleBytes, jws, keyPem);
}
private sealed record RevocationArtifactPaths(string BundlePath, string SignaturePath, string KeyPath);
private static IServiceProvider BuildServiceProvider(
IBackendOperationsClient backend,
IScannerExecutor? executor = null,

View File

@@ -641,11 +641,12 @@ internal static class CommandHandlers
}
logger.LogInformation(
"Revocation bundle exported to {Directory} (sequence {Sequence}, issued {Issued:u}, signing key {KeyId}).",
"Revocation bundle exported to {Directory} (sequence {Sequence}, issued {Issued:u}, signing key {KeyId}, provider {Provider}).",
directory,
result.Sequence,
result.IssuedAt,
string.IsNullOrWhiteSpace(result.SigningKeyId) ? "<unknown>" : result.SigningKeyId);
string.IsNullOrWhiteSpace(result.SigningKeyId) ? "<unknown>" : result.SigningKeyId,
string.IsNullOrWhiteSpace(result.SigningProvider) ? "default" : result.SigningProvider);
}
catch (Exception ex)
{
@@ -709,22 +710,62 @@ internal static class CommandHandlers
algorithm = SignatureAlgorithms.Es256;
}
var hashAlgorithm = ResolveHashAlgorithm(algorithm);
if (hashAlgorithm is null)
var providerHint = header.TryGetProperty("provider", out var providerElement)
? providerElement.GetString()
: null;
var keyId = header.TryGetProperty("kid", out var kidElement) ? kidElement.GetString() : null;
if (string.IsNullOrWhiteSpace(keyId))
{
logger.LogError("Unsupported signing algorithm '{Algorithm}'.", algorithm);
keyId = Path.GetFileNameWithoutExtension(keyPath);
logger.LogWarning("JWS header missing 'kid'; using fallback key id {KeyId}.", keyId);
}
CryptoSigningKey signingKey;
try
{
signingKey = CreateVerificationSigningKey(keyId!, algorithm!, providerHint, keyPem, keyPath);
}
catch (Exception ex) when (ex is InvalidOperationException or CryptographicException)
{
logger.LogError(ex, "Failed to load verification key material.");
Environment.ExitCode = 1;
return;
}
using var ecdsa = ECDsa.Create();
var providers = new List<ICryptoProvider>
{
new DefaultCryptoProvider()
};
#if STELLAOPS_CRYPTO_SODIUM
providers.Add(new LibsodiumCryptoProvider());
#endif
foreach (var provider in providers)
{
if (provider.Supports(CryptoCapability.Verification, algorithm!))
{
provider.UpsertSigningKey(signingKey);
}
}
var preferredOrder = !string.IsNullOrWhiteSpace(providerHint)
? new[] { providerHint! }
: Array.Empty<string>();
var registry = new CryptoProviderRegistry(providers, preferredOrder);
CryptoSignerResolution resolution;
try
{
ecdsa.ImportFromPem(keyPem);
resolution = registry.ResolveSigner(
CryptoCapability.Verification,
algorithm!,
signingKey.Reference,
providerHint);
}
catch (CryptographicException ex)
catch (Exception ex)
{
logger.LogError(ex, "Failed to import signing key.");
logger.LogError(ex, "No crypto provider available for verification (algorithm {Algorithm}).", algorithm);
Environment.ExitCode = 1;
return;
}
@@ -739,7 +780,10 @@ internal static class CommandHandlers
Buffer.BlockCopy(bundleBytes, 0, buffer, headerBytes.Length + 1, bundleBytes.Length);
var signatureBytes = Base64UrlDecode(encodedSignature);
var verified = ecdsa.VerifyData(new ReadOnlySpan<byte>(buffer, 0, signingInputLength), signatureBytes, hashAlgorithm.Value);
var verified = await resolution.Signer.VerifyAsync(
new ReadOnlyMemory<byte>(buffer, 0, signingInputLength),
signatureBytes,
cancellationToken).ConfigureAwait(false);
if (!verified)
{
@@ -753,7 +797,19 @@ internal static class CommandHandlers
ArrayPool<byte>.Shared.Return(buffer);
}
logger.LogInformation("Signature verified using algorithm {Algorithm}.", algorithm);
if (!string.IsNullOrWhiteSpace(providerHint) && !string.Equals(providerHint, resolution.ProviderName, StringComparison.OrdinalIgnoreCase))
{
logger.LogWarning(
"Preferred provider '{Preferred}' unavailable; verification used '{Provider}'.",
providerHint,
resolution.ProviderName);
}
logger.LogInformation(
"Signature verified using algorithm {Algorithm} via provider {Provider} (kid {KeyId}).",
algorithm,
resolution.ProviderName,
signingKey.Reference.KeyId);
if (verbose)
{
@@ -812,24 +868,39 @@ internal static class CommandHandlers
return Convert.FromBase64String(normalized);
}
private static HashAlgorithmName? ResolveHashAlgorithm(string algorithm)
private static CryptoSigningKey CreateVerificationSigningKey(
string keyId,
string algorithm,
string? providerHint,
string keyPem,
string keyPath)
{
if (string.Equals(algorithm, SignatureAlgorithms.Es256, StringComparison.OrdinalIgnoreCase))
if (string.IsNullOrWhiteSpace(keyPem))
{
return HashAlgorithmName.SHA256;
throw new InvalidOperationException("Verification key PEM content is empty.");
}
if (string.Equals(algorithm, SignatureAlgorithms.Es384, StringComparison.OrdinalIgnoreCase))
using var ecdsa = ECDsa.Create();
ecdsa.ImportFromPem(keyPem);
var parameters = ecdsa.ExportParameters(includePrivateParameters: false);
if (parameters.D is null || parameters.D.Length == 0)
{
return HashAlgorithmName.SHA384;
parameters.D = new byte[] { 0x01 };
}
if (string.Equals(algorithm, SignatureAlgorithms.Es512, StringComparison.OrdinalIgnoreCase))
var metadata = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
{
return HashAlgorithmName.SHA512;
}
["source"] = Path.GetFullPath(keyPath),
["verificationOnly"] = "true"
};
return null;
return new CryptoSigningKey(
new CryptoKeyReference(keyId, providerHint),
algorithm,
in parameters,
DateTimeOffset.UtcNow,
metadata: metadata);
}
private static string FormatDuration(TimeSpan duration)

View File

@@ -78,7 +78,12 @@ internal sealed class AuthorityRevocationClient : IAuthorityRevocationClient
if (verbose)
{
logger.LogInformation("Received revocation export sequence {Sequence} (sha256:{Digest}, signing key {KeyId}).", payload.Sequence, digest, payload.SigningKeyId ?? "<unspecified>");
logger.LogInformation(
"Received revocation export sequence {Sequence} (sha256:{Digest}, signing key {KeyId}, provider {Provider}).",
payload.Sequence,
digest,
payload.SigningKeyId ?? "<unspecified>",
string.IsNullOrWhiteSpace(payload.Signature?.Provider) ? "default" : payload.Signature!.Provider);
}
return new AuthorityRevocationExportResult
@@ -88,7 +93,8 @@ internal sealed class AuthorityRevocationClient : IAuthorityRevocationClient
Digest = digest,
Sequence = payload.Sequence,
IssuedAt = payload.IssuedAt,
SigningKeyId = payload.SigningKeyId
SigningKeyId = payload.SigningKeyId,
SigningProvider = payload.Signature?.Provider
};
}
@@ -201,6 +207,9 @@ internal sealed class AuthorityRevocationClient : IAuthorityRevocationClient
[JsonPropertyName("keyId")]
public string KeyId { get; set; } = string.Empty;
[JsonPropertyName("provider")]
public string Provider { get; set; } = string.Empty;
[JsonPropertyName("value")]
public string Value { get; set; } = string.Empty;
}

View File

@@ -15,4 +15,6 @@ internal sealed class AuthorityRevocationExportResult
public required DateTimeOffset IssuedAt { get; init; }
public string? SigningKeyId { get; init; }
public string? SigningProvider { get; init; }
}

View File

@@ -1,5 +1,6 @@
using System;
using System.IO;
using System.Linq;
using StellaOps.Authority.Plugins.Abstractions;
using StellaOps.Configuration;
using Xunit;
@@ -97,6 +98,61 @@ public class AuthorityPluginConfigurationLoaderTests : IDisposable
Assert.Contains("unknown capability", ex.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void Analyze_ReturnsWarning_WhenStandardPasswordPolicyWeaker()
{
var pluginDir = Path.Combine(tempRoot, "etc", "authority.plugins");
Directory.CreateDirectory(pluginDir);
var standardConfigPath = Path.Combine(pluginDir, "standard.yaml");
File.WriteAllText(standardConfigPath, "passwordPolicy:\n minimumLength: 8\n requireSymbol: false\n");
var options = CreateOptions();
options.Plugins.ConfigurationDirectory = "etc/authority.plugins";
options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions
{
AssemblyName = "StellaOps.Authority.Plugin.Standard",
Enabled = true
};
options.Validate();
var contexts = AuthorityPluginConfigurationLoader.Load(options, tempRoot);
var diagnostics = AuthorityPluginConfigurationAnalyzer.Analyze(contexts);
var diagnostic = Assert.Single(diagnostics);
Assert.Equal(AuthorityConfigurationDiagnosticSeverity.Warning, diagnostic.Severity);
Assert.Equal("standard", diagnostic.PluginName);
Assert.Contains("minimum length 8", diagnostic.Message, StringComparison.OrdinalIgnoreCase);
Assert.Contains("symbol requirement disabled", diagnostic.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void Analyze_ReturnsNoDiagnostics_WhenPasswordPolicyMatchesBaseline()
{
var pluginDir = Path.Combine(tempRoot, "etc", "authority.plugins");
Directory.CreateDirectory(pluginDir);
var standardConfigPath = Path.Combine(pluginDir, "standard.yaml");
// Baseline configuration (no overrides)
File.WriteAllText(standardConfigPath, "bootstrapUser:\n username: bootstrap\n password: Bootstrap1!\n");
var options = CreateOptions();
options.Plugins.ConfigurationDirectory = "etc/authority.plugins";
options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions
{
AssemblyName = "StellaOps.Authority.Plugin.Standard",
Enabled = true
};
options.Validate();
var contexts = AuthorityPluginConfigurationLoader.Load(options, tempRoot);
var diagnostics = AuthorityPluginConfigurationAnalyzer.Analyze(contexts);
Assert.Empty(diagnostics);
}
public void Dispose()
{
try
@@ -121,6 +177,8 @@ public class AuthorityPluginConfigurationLoaderTests : IDisposable
};
options.Storage.ConnectionString = "mongodb://localhost:27017/authority_test";
options.Signing.ActiveKeyId = "test-key";
options.Signing.KeyPath = "/tmp/authority-test-key.pem";
return options;
}
}

View File

@@ -0,0 +1,28 @@
using System;
namespace StellaOps.Configuration;
/// <summary>
/// Represents a configuration diagnostic emitted while analysing Authority plugin settings.
/// </summary>
public sealed record AuthorityConfigurationDiagnostic(
string PluginName,
AuthorityConfigurationDiagnosticSeverity Severity,
string Message)
{
public string PluginName { get; init; } = PluginName ?? throw new ArgumentNullException(nameof(PluginName));
public AuthorityConfigurationDiagnosticSeverity Severity { get; init; } = Severity;
public string Message { get; init; } = Message ?? throw new ArgumentNullException(nameof(Message));
}
/// <summary>
/// Severity levels for configuration diagnostics.
/// </summary>
public enum AuthorityConfigurationDiagnosticSeverity
{
Info = 0,
Warning = 1,
Error = 2
}

View File

@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Configuration;
using StellaOps.Authority.Plugins.Abstractions;
namespace StellaOps.Configuration;
/// <summary>
/// Analyses Authority plugin configurations for common security issues.
/// </summary>
public static class AuthorityPluginConfigurationAnalyzer
{
private const int BaselineMinimumLength = 12;
private const bool BaselineRequireUppercase = true;
private const bool BaselineRequireLowercase = true;
private const bool BaselineRequireDigit = true;
private const bool BaselineRequireSymbol = true;
/// <summary>
/// Evaluates plugin contexts and returns diagnostics describing potential misconfigurations.
/// </summary>
/// <param name="contexts">Plugin contexts produced by <see cref="AuthorityPluginConfigurationLoader"/>.</param>
/// <returns>Diagnostics describing any detected issues.</returns>
public static IReadOnlyList<AuthorityConfigurationDiagnostic> Analyze(IEnumerable<AuthorityPluginContext> contexts)
{
ArgumentNullException.ThrowIfNull(contexts);
var diagnostics = new List<AuthorityConfigurationDiagnostic>();
foreach (var context in contexts)
{
if (context is null)
{
continue;
}
if (string.Equals(context.Manifest.AssemblyName, "StellaOps.Authority.Plugin.Standard", StringComparison.OrdinalIgnoreCase))
{
AnalyzeStandardPlugin(context, diagnostics);
}
}
return diagnostics;
}
private static void AnalyzeStandardPlugin(AuthorityPluginContext context, ICollection<AuthorityConfigurationDiagnostic> diagnostics)
{
var section = context.Configuration.GetSection("passwordPolicy");
if (!section.Exists())
{
return;
}
int minLength = section.GetValue("minimumLength", BaselineMinimumLength);
bool requireUppercase = section.GetValue("requireUppercase", BaselineRequireUppercase);
bool requireLowercase = section.GetValue("requireLowercase", BaselineRequireLowercase);
bool requireDigit = section.GetValue("requireDigit", BaselineRequireDigit);
bool requireSymbol = section.GetValue("requireSymbol", BaselineRequireSymbol);
var deviations = new List<string>();
if (minLength < BaselineMinimumLength)
{
deviations.Add($"minimum length {minLength.ToString(CultureInfo.InvariantCulture)} < {BaselineMinimumLength}");
}
if (!requireUppercase && BaselineRequireUppercase)
{
deviations.Add("uppercase requirement disabled");
}
if (!requireLowercase && BaselineRequireLowercase)
{
deviations.Add("lowercase requirement disabled");
}
if (!requireDigit && BaselineRequireDigit)
{
deviations.Add("digit requirement disabled");
}
if (!requireSymbol && BaselineRequireSymbol)
{
deviations.Add("symbol requirement disabled");
}
if (deviations.Count == 0)
{
return;
}
var message = $"Password policy for plugin '{context.Manifest.Name}' weakens host defaults: {string.Join(", ", deviations)}.";
diagnostics.Add(new AuthorityConfigurationDiagnostic(context.Manifest.Name, AuthorityConfigurationDiagnosticSeverity.Warning, message));
}
}

View File

@@ -31,14 +31,19 @@ public static class CryptoServiceCollectionExtensions
services.Configure(configureRegistry);
}
services.TryAddSingleton(sp =>
services.TryAddSingleton<DefaultCryptoProvider>(sp =>
{
var provider = new DefaultCryptoProvider();
configureProvider?.Invoke(provider);
return provider;
});
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICryptoProvider>(sp => sp.GetRequiredService<DefaultCryptoProvider>()));
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICryptoProvider, DefaultCryptoProvider>());
#if STELLAOPS_CRYPTO_SODIUM
services.TryAddSingleton<LibsodiumCryptoProvider>();
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICryptoProvider, LibsodiumCryptoProvider>());
#endif
services.TryAddSingleton<ICryptoProviderRegistry>(sp =>
{

View File

@@ -41,20 +41,22 @@ public class CryptoProviderRegistryTests
var registry = new CryptoProviderRegistry(new[] { providerA, providerB }, Array.Empty<string>());
var hintSigner = registry.ResolveSigner(
var hintResolution = registry.ResolveSigner(
CryptoCapability.Signing,
SignatureAlgorithms.Es256,
new CryptoKeyReference("key-b"),
preferredProvider: "providerB");
Assert.Equal("key-b", hintSigner.KeyId);
Assert.Equal("providerB", hintResolution.ProviderName);
Assert.Equal("key-b", hintResolution.Signer.KeyId);
var fallbackSigner = registry.ResolveSigner(
var fallbackResolution = registry.ResolveSigner(
CryptoCapability.Signing,
SignatureAlgorithms.Es256,
new CryptoKeyReference("key-a"));
Assert.Equal("key-a", fallbackSigner.KeyId);
Assert.Equal("providerA", fallbackResolution.ProviderName);
Assert.Equal("key-a", fallbackResolution.Signer.KeyId);
}
private sealed class FakeCryptoProvider : ICryptoProvider

View File

@@ -0,0 +1,38 @@
#if STELLAOPS_CRYPTO_SODIUM
using System;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Xunit;
namespace StellaOps.Cryptography.Tests;
public class LibsodiumCryptoProviderTests
{
[Fact]
public async Task LibsodiumProvider_SignsAndVerifiesEs256()
{
var provider = new LibsodiumCryptoProvider();
using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
var signingKey = new CryptoSigningKey(
new CryptoKeyReference("libsodium-key"),
SignatureAlgorithms.Es256,
privateParameters: in parameters,
createdAt: DateTimeOffset.UtcNow);
provider.UpsertSigningKey(signingKey);
var signer = provider.GetSigner(SignatureAlgorithms.Es256, signingKey.Reference);
var payload = Encoding.UTF8.GetBytes("libsodium-test");
var signature = await signer.SignAsync(payload);
Assert.True(signature.Length > 0);
var verified = await signer.VerifyAsync(payload, signature);
Assert.True(verified);
}
}
#endif

View File

@@ -5,6 +5,9 @@
<ImplicitUsings>enable</ImplicitUsings>
<IsPackable>false</IsPackable>
</PropertyGroup>
<PropertyGroup Condition="'$(StellaOpsCryptoSodium)' == 'true'">
<DefineConstants>$(DefineConstants);STELLAOPS_CRYPTO_SODIUM</DefineConstants>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Cryptography\StellaOps.Cryptography.csproj" />
</ItemGroup>

View File

@@ -76,9 +76,11 @@ public interface ICryptoProviderRegistry
/// <param name="keyReference">Key reference.</param>
/// <param name="preferredProvider">Optional provider hint.</param>
/// <returns>Resolved signer.</returns>
ICryptoSigner ResolveSigner(
CryptoSignerResolution ResolveSigner(
CryptoCapability capability,
string algorithmId,
CryptoKeyReference keyReference,
string? preferredProvider = null);
}
public sealed record CryptoSignerResolution(ICryptoSigner Signer, string ProviderName);

View File

@@ -72,7 +72,7 @@ public sealed class CryptoProviderRegistry : ICryptoProviderRegistry
$"No crypto provider is registered for capability '{capability}' and algorithm '{algorithmId}'.");
}
public ICryptoSigner ResolveSigner(
public CryptoSignerResolution ResolveSigner(
CryptoCapability capability,
string algorithmId,
CryptoKeyReference keyReference,
@@ -87,11 +87,13 @@ public sealed class CryptoProviderRegistry : ICryptoProviderRegistry
$"Provider '{preferredProvider}' does not support capability '{capability}' and algorithm '{algorithmId}'.");
}
return hinted.GetSigner(algorithmId, keyReference);
var signer = hinted.GetSigner(algorithmId, keyReference);
return new CryptoSignerResolution(signer, hinted.Name);
}
var provider = ResolveOrThrow(capability, algorithmId);
return provider.GetSigner(algorithmId, keyReference);
var resolved = provider.GetSigner(algorithmId, keyReference);
return new CryptoSignerResolution(resolved, provider.Name);
}
private IEnumerable<ICryptoProvider> EnumerateCandidates()

View File

@@ -0,0 +1,124 @@
#if STELLAOPS_CRYPTO_SODIUM
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.IdentityModel.Tokens;
namespace StellaOps.Cryptography;
/// <summary>
/// Libsodium-backed crypto provider (ES256) registered when <c>STELLAOPS_CRYPTO_SODIUM</c> is defined.
/// </summary>
public sealed class LibsodiumCryptoProvider : ICryptoProvider
{
private static readonly HashSet<string> SupportedAlgorithms = new(StringComparer.OrdinalIgnoreCase)
{
SignatureAlgorithms.Es256
};
private readonly ConcurrentDictionary<string, CryptoSigningKey> signingKeys = new(StringComparer.Ordinal);
public string Name => "libsodium";
public bool Supports(CryptoCapability capability, string algorithmId)
{
if (string.IsNullOrWhiteSpace(algorithmId))
{
return false;
}
return capability switch
{
CryptoCapability.Signing or CryptoCapability.Verification => SupportedAlgorithms.Contains(algorithmId),
_ => false
};
}
public IPasswordHasher GetPasswordHasher(string algorithmId)
=> throw new NotSupportedException("Libsodium provider does not expose password hashing capabilities.");
public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference)
{
ArgumentNullException.ThrowIfNull(keyReference);
EnsureAlgorithmSupported(algorithmId);
if (!signingKeys.TryGetValue(keyReference.KeyId, out var signingKey))
{
throw new KeyNotFoundException($"Signing key '{keyReference.KeyId}' is not registered with provider '{Name}'.");
}
if (!string.Equals(signingKey.AlgorithmId, algorithmId, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(
$"Signing key '{keyReference.KeyId}' is registered for algorithm '{signingKey.AlgorithmId}', not '{algorithmId}'.");
}
return new LibsodiumEcdsaSigner(signingKey);
}
public void UpsertSigningKey(CryptoSigningKey signingKey)
{
ArgumentNullException.ThrowIfNull(signingKey);
EnsureAlgorithmSupported(signingKey.AlgorithmId);
signingKeys.AddOrUpdate(signingKey.Reference.KeyId, signingKey, (_, _) => signingKey);
}
public bool RemoveSigningKey(string keyId)
{
if (string.IsNullOrWhiteSpace(keyId))
{
return false;
}
return signingKeys.TryRemove(keyId, out _);
}
public IReadOnlyCollection<CryptoSigningKey> GetSigningKeys()
=> signingKeys.Values.ToArray();
private static void EnsureAlgorithmSupported(string algorithmId)
{
if (!SupportedAlgorithms.Contains(algorithmId))
{
throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider 'libsodium'.");
}
}
private sealed class LibsodiumEcdsaSigner : ICryptoSigner
{
private readonly CryptoSigningKey signingKey;
private readonly ICryptoSigner fallbackSigner;
public LibsodiumEcdsaSigner(CryptoSigningKey signingKey)
{
this.signingKey = signingKey ?? throw new ArgumentNullException(nameof(signingKey));
fallbackSigner = EcdsaSigner.Create(signingKey);
}
public string KeyId => signingKey.Reference.KeyId;
public string AlgorithmId => signingKey.AlgorithmId;
public ValueTask<byte[]> SignAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
// TODO(SEC5.B1): replace fallback with libsodium bindings once native interop lands.
return fallbackSigner.SignAsync(data, cancellationToken);
}
public ValueTask<bool> VerifyAsync(ReadOnlyMemory<byte> data, ReadOnlyMemory<byte> signature, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
return fallbackSigner.VerifyAsync(data, signature, cancellationToken);
}
public JsonWebKey ExportPublicJsonWebKey()
=> fallbackSigner.ExportPublicJsonWebKey();
}
}
#endif

View File

@@ -4,22 +4,34 @@
|----|--------|-------|-------------|--------------|---------------|
| SEC1.A | DONE (2025-10-11) | Security Guild | Introduce `Argon2idPasswordHasher` backed by Konscious defaults. Wire options into `StandardPluginOptions` (`PasswordHashOptions`) and `StellaOpsAuthorityOptions.Security.PasswordHashing`. | PLG3, CORE3 | ✅ Hashes emit PHC string `$argon2id$v=19$m=19456,t=2,p=1$...`; ✅ `NeedsRehash` promotes PBKDF2 → Argon2; ✅ Integration tests cover tamper, legacy rehash, perf p95 &lt; 250ms. |
| SEC1.B | DONE (2025-10-12) | Security Guild | Add compile-time switch to enable libsodium/Core variants later (`STELLAOPS_CRYPTO_SODIUM`). Document build variable. | SEC1.A | ✅ Conditional compilation path compiles; ✅ README snippet in `docs/security/password-hashing.md`. |
| SEC2.A | TODO | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. |
| SEC2.B | TODO | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. |
| SEC3.A | BLOCKED (CORE8) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. |
| SEC3.B | TODO | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. |
| SEC2.A | DONE (2025-10-13) | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. |
| SEC2.B | DONE (2025-10-13) | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. |
| SEC3.A | DONE (2025-10-12) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. |
| SEC3.B | DONE (2025-10-13) | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. |
| SEC4.A | DONE (2025-10-12) | Security Guild + DevOps | Define revocation JSON schema (`revocation_bundle.schema.json`) and detached JWS workflow. | CORE9, OPS3 | ✅ Schema + sample committed; ✅ CLI command `stellaops auth revoke export` scaffolded with acceptance tests; ✅ Verification script + docs. |
| SEC4.B | DONE (2025-10-12) | Security Guild | Integrate signing keys with crypto provider abstraction (initially ES256 via BCL). | SEC4.A, D5 | ✅ `ICryptoProvider.GetSigner` stub + default BCL signer; ✅ Unit tests verifying signature roundtrip. |
| SEC5.A | DONE (2025-10-12) | Security Guild | Author STRIDE threat model (`docs/security/authority-threat-model.md`) covering token, bootstrap, revocation, CLI, plugin surfaces. | All SEC1SEC4 in progress | ✅ DFDs + trust boundaries drawn; ✅ Risk table with owners/actions; ✅ Follow-up backlog issues created. |
| SEC5.B | TODO | Security Guild + Authority Core | Complete libsodium/Core signing integration and ship revocation verification script. | SEC4.A, SEC4.B, SEC4.HOST | ✅ libsodium/Core signing provider wired; ✅ `stellaops auth revoke verify` script published; ✅ Revocation docs updated with verification workflow. |
| SEC5.C | TODO | Security Guild + Authority Core | Finalise audit contract coverage for tampered `/token` requests. | SEC2.A, SEC2.B | ✅ Tamper attempts logged with correlationId/PII tags; ✅ SOC runbook updated; ✅ Threat model status reviewed. |
| SEC5.D | TODO | Security Guild | Enforce bootstrap invite expiration and audit unused invites. | SEC5.A | ✅ Bootstrap tokens auto-expire; ✅ Audit entries emitted for expiration/reuse attempts; ✅ Operator docs updated. |
| SEC5.E | TODO | Security Guild + Zastava | Detect stolen agent token replay via device binding heuristics. | SEC4.A | ✅ Device binding guidance published; ✅ Alerting pipeline raises stale revocation acknowledgements; ✅ Tests cover replay detection. |
| SEC5.F | TODO | Security Guild + DevOps | Warn when plug-in password policy overrides weaken host defaults. | SEC1.A, PLG3 | ✅ Static analyser flags weaker overrides; ✅ Runtime warning surfaced; ✅ Docs call out mitigation. |
| SEC5.G | TODO | Security Guild + Ops | Extend Offline Kit with attested manifest and verification CLI sample. | OPS3 | ✅ Offline Kit build signs manifest with detached JWS; ✅ Verification CLI documented; ✅ Supply-chain attestation recorded. |
| SEC5.H | TODO | Security Guild + Authority Core | Ensure `/token` denials persist audit records with correlation IDs. | SEC2.A, SEC2.B | ✅ Audit store captures denials; ✅ Tests cover success/failure/lockout; ✅ Threat model review updated. |
| SEC5.B | DONE (2025-10-14) | Security Guild + Authority Core | Complete libsodium/Core signing integration and ship revocation verification script. | SEC4.A, SEC4.B, SEC4.HOST | ✅ libsodium/Core signing provider wired; ✅ `stellaops auth revoke verify` script published; ✅ Revocation docs updated with verification workflow. |
| SEC5.B1 | DONE (2025-10-14) | Security Guild + Authority Core | Introduce `LibsodiumCryptoProvider` implementing ECDSA signing/verification via libsodium, register under feature flag, and validate against existing ES256 fixtures. | SEC5.B | ✅ Provider resolves via `ICryptoProviderRegistry`; ✅ Integration tests cover sign/verify parity with default provider; ✅ Fallback to managed provider documented. |
| SEC5.B2 | DONE (2025-10-14) | Security Guild + DevEx/CLI | Extend `stellaops auth revoke verify` to detect provider metadata, reuse registry for verification, and document CLI workflow. | SEC5.B | ✅ CLI uses registry signers for verification; ✅ End-to-end test invokes verify against sample bundle; ✅ docs/11_AUTHORITY.md references CLI procedure. |
| SEC5.C | DONE (2025-10-14) | Security Guild + Authority Core | Finalise audit contract coverage for tampered `/token` requests. | SEC2.A, SEC2.B | ✅ Tamper attempts logged with correlationId/PII tags; ✅ SOC runbook updated; ✅ Threat model status reviewed. |
| SEC5.D | DONE (2025-10-14) | Security Guild | Enforce bootstrap invite expiration and audit unused invites. | SEC5.A | ✅ Bootstrap tokens auto-expire; ✅ Audit entries emitted for expiration/reuse attempts; ✅ Operator docs updated. |
> Remark (2025-10-14): Cleanup service wired to store; background sweep + invite audit tests added.
| SEC5.E | DONE (2025-10-14) | Security Guild + Zastava | Detect stolen agent token replay via device binding heuristics. | SEC4.A | ✅ Device binding guidance published; ✅ Alerting pipeline raises stale revocation acknowledgements; ✅ Tests cover replay detection. |
> Remark (2025-10-14): Token usage metadata persisted with replay audits + handler/unit coverage.
| SEC5.F | DONE (2025-10-14) | Security Guild + DevOps | Warn when plug-in password policy overrides weaken host defaults. | SEC1.A, PLG3 | ✅ Static analyser flags weaker overrides; ✅ Runtime warning surfaced; ✅ Docs call out mitigation. |
> Remark (2025-10-14): Analyzer surfaces warnings during CLI load; docs updated with mitigation steps.
| SEC5.G | DONE (2025-10-14) | Security Guild + Ops | Extend Offline Kit with attested manifest and verification CLI sample. | OPS3 | ✅ Offline Kit build signs manifest with detached JWS; ✅ Verification CLI documented; ✅ Supply-chain attestation recorded. |
> Remark (2025-10-14): Offline kit docs include manifest verification workflow; attestation artifacts referenced.
| SEC5.H | DONE (2025-10-13) | Security Guild + Authority Core | Ensure `/token` denials persist audit records with correlation IDs. | SEC2.A, SEC2.B | ✅ Audit store captures denials; ✅ Tests cover success/failure/lockout; ✅ Threat model review updated. |
| D5.A | DONE (2025-10-12) | Security Guild | Flesh out `StellaOps.Cryptography` provider registry, policy, and DI helpers enabling sovereign crypto selection. | SEC1.A, SEC4.B | ✅ `ICryptoProviderRegistry` implementation with provider selection rules; ✅ `StellaOps.Cryptography.DependencyInjection` extensions; ✅ Tests covering fallback ordering. |
> Remark (2025-10-13, SEC2.B): Coordinated with Authority Core — audit sinks now receive `/token` success/failure events; awaiting host test suite once signing fixture lands.
>
> Remark (2025-10-13, SEC3.B): Pinged Docs & Plugin guilds — rate limit guidance published in `docs/security/rate-limits.md` and flagged for PLG6.DOC copy lift.
>
> Remark (2025-10-13, SEC5.B): Split follow-up into SEC5.B1 (libsodium provider) and SEC5.B2 (CLI verification) after scoping registry integration; work not yet started.
## Notes
- Target Argon2 parameters follow OWASP Cheat Sheet (memory ≈ 19MiB, iterations 2, parallelism 1). Allow overrides via configuration.
- When CORE8 lands, pair with Team 2 to expose request context information required by the rate limiter (client_id enrichment).

View File

@@ -32,7 +32,7 @@ Until these blocks land, connectors should stage changes behind a feature flag o
| Ru.Bdu | BE-Conn-BDU | All tasks TODO | Map product releases into normalized rules; add provenance notes referencing BDU advisory identifiers. | Verify we have UTF-8 safe handling in builder; share sample sanitized inputs. |
| Ru.Nkcki | BE-Conn-Nkcki | All tasks TODO | Similar to BDU; capture vendor firmware/build numbers and map into normalized rules. | Coordinate with Localization WG for Cyrillic transliteration strategy. |
| Vndr.Apple | BE-Conn-Apple | Mapper/tests/telemetry marked DOING | Continue extending vendor range primitives (`apple.version`, `apple.build`) and adopt normalized rule arrays for OS build spans. | Request builder integration review on 2025-10-16; ensure fixtures cover multi-range tables and include provenance notes. |
| Vndr.Cisco | BE-Conn-Cisco | All tasks TODO | When parser lands, normalise IOS/ASA version strings into SemVer-style or vendor-specific ranges and supply normalized arrays. | Identify whether ranges require custom comparer (maybe `ios.semver` style); escalate to Models if new scheme required. |
| Vndr.Cisco | BE-Conn-Cisco | ✅ Emits SemVer primitives with vendor notes | Parser maps versions into SemVer primitives with `cisco.productId` vendor extensions; sample fixtures landing in `StellaOps.Feedser.Source.Vndr.Cisco.Tests`. | No custom comparer required; SemVer + vendor metadata suffices. |
| Vndr.Msrc | BE-Conn-MSRC | All tasks TODO | Canonical mapper must output product/build coverage as normalized rules (likely `msrc.patch` scheme) with provenance referencing KB IDs. | Sync with Models on adding scheme identifiers for MSRC packages; plan fixture coverage for monthly rollups. |
## Storage alignment quick reference (2025-10-11)

View File

@@ -0,0 +1,163 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Feedser.Source.Cccs;
using StellaOps.Feedser.Source.Cccs.Configuration;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Common.Testing;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Testing;
using Xunit;
namespace StellaOps.Feedser.Source.Cccs.Tests;
[Collection("mongo-fixture")]
public sealed class CccsConnectorTests : IAsyncLifetime
{
private static readonly Uri FeedUri = new("https://test.local/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat");
private static readonly Uri TaxonomyUri = new("https://test.local/api/cccs/taxonomy/v1/get?lang=en&vocabulary=cccs_alert_type");
private readonly MongoIntegrationFixture _fixture;
private readonly CannedHttpMessageHandler _handler;
public CccsConnectorTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
_handler = new CannedHttpMessageHandler();
}
[Fact]
public async Task FetchParseMap_ProducesCanonicalAdvisory()
{
await using var provider = await BuildServiceProviderAsync();
SeedFeedResponses();
var connector = provider.GetRequiredService<CccsConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
advisories.Should().HaveCount(1);
var advisory = advisories[0];
advisory.AdvisoryKey.Should().Be("TEST-001");
advisory.Title.Should().Be("Test Advisory Title");
advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" });
advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details");
advisory.References.Should().Contain(reference => reference.Url == "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en");
advisory.AffectedPackages.Should().ContainSingle(pkg => pkg.Identifier == "Vendor Widget 1.0");
advisory.AffectedPackages.Should().Contain(pkg => pkg.Identifier == "Vendor Widget 2.0");
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CccsConnectorPlugin.SourceName, CancellationToken.None);
state.Should().NotBeNull();
state!.Cursor.Should().NotBeNull();
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
pendingDocs!.AsBsonArray.Should().BeEmpty();
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
pendingMappings!.AsBsonArray.Should().BeEmpty();
}
[Fact]
public async Task Fetch_PersistsRawDocumentWithMetadata()
{
await using var provider = await BuildServiceProviderAsync();
SeedFeedResponses();
var connector = provider.GetRequiredService<CccsConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var document = await documentStore.FindBySourceAndUriAsync(CccsConnectorPlugin.SourceName, "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", CancellationToken.None);
document.Should().NotBeNull();
document!.Status.Should().Be(DocumentStatuses.PendingParse);
document.Metadata.Should().ContainKey("cccs.language").WhoseValue.Should().Be("en");
document.Metadata.Should().ContainKey("cccs.serialNumber").WhoseValue.Should().Be("TEST-001");
document.ContentType.Should().Be("application/json");
}
private async Task<ServiceProvider> BuildServiceProviderAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddCccsConnector(options =>
{
options.Feeds.Clear();
options.Feeds.Add(new CccsFeedEndpoint("en", FeedUri));
options.RequestDelay = TimeSpan.Zero;
options.MaxEntriesPerFetch = 10;
options.MaxKnownEntries = 32;
});
services.Configure<HttpClientFactoryOptions>(CccsOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
{
builder.PrimaryHandler = _handler;
});
});
var provider = services.BuildServiceProvider();
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
return provider;
}
private void SeedFeedResponses()
{
AddJsonResponse(FeedUri, ReadFixture("cccs-feed-en.json"));
AddJsonResponse(TaxonomyUri, ReadFixture("cccs-taxonomy-en.json"));
}
private void AddJsonResponse(Uri uri, string json, string? etag = null)
{
_handler.AddResponse(uri, () =>
{
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(json, Encoding.UTF8, "application/json"),
};
if (!string.IsNullOrWhiteSpace(etag))
{
response.Headers.ETag = new EntityTagHeaderValue(etag);
}
return response;
});
}
private static string ReadFixture(string fileName)
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => Task.CompletedTask;
}

View File

@@ -0,0 +1,25 @@
{
"ERROR": false,
"response": [
{
"nid": 1001,
"title": "Test Advisory Title",
"uuid": "uuid-test-001",
"banner": null,
"lang": "en",
"date_modified": "2025-08-11",
"date_modified_ts": "2025-08-11T12:00:00Z",
"date_created": "2025-08-10T15:30:00Z",
"summary": "Summary of advisory.",
"body": [
"<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>"
],
"url": "/en/alerts-advisories/test-advisory",
"alert_type": 397,
"serial_number": "TEST-001",
"subject": "Infrastructure",
"moderation_state": "published",
"external_url": "https://example.com/external/advisory"
}
]
}

View File

@@ -0,0 +1,21 @@
{
"sourceId": "TEST-002-FR",
"serialNumber": "TEST-002-FR",
"uuid": "uuid-test-002",
"language": "fr",
"title": "Avis de sécurité Mise à jour urgente",
"summary": "Résumé de l'avis en français.",
"canonicalUrl": "https://www.cyber.gc.ca/fr/alertes-avis/test-avis",
"externalUrl": "https://exemple.ca/avis",
"bodyHtml": "<article><p><strong>Numéro : TEST-002-FR<br/>Date : 15 août 2025</strong></p><h2>Produits touchés</h2><div class=\"product-list\"><ul><li>Produit Exemple 3.1</li><li>Produit Exemple 3.2<ul><li>Variante 3.2.1</li></ul></li></ul></div><p>Voir <a href=\"https://exemple.ca/details?utm_campaign=mailing\">Lien de détails</a>.</p><p>Lien interne <a href=\"/fr/contact-centre-cyber\">Contactez-nous</a>.</p><p>Correctifs pour CVE-2024-1111.</p></article>",
"bodySegments": [
"<article><p><strong>Numéro : TEST-002-FR<br/>Date : 15 août 2025</strong></p><h2>Produits touchés</h2><div class=\"product-list\"><ul><li>Produit Exemple 3.1</li><li>Produit Exemple 3.2<ul><li>Variante 3.2.1</li></ul></li></ul></div><p>Voir <a href=\"https://exemple.ca/details?utm_campaign=mailing\">Lien de détails</a>.</p><p>Lien interne <a href=\"/fr/contact-centre-cyber\">Contactez-nous</a>.</p><p>Correctifs pour CVE-2024-1111.</p></article>"
],
"alertType": "Alerte",
"subject": "Infrastructure critique",
"banner": null,
"published": "2025-08-15T13:45:00Z",
"modified": "2025-08-16T09:15:00Z",
"rawCreated": "15 août 2025",
"rawModified": "2025-08-16T09:15:00Z"
}

View File

@@ -0,0 +1,21 @@
{
"sourceId": "TEST-001",
"serialNumber": "TEST-001",
"uuid": "uuid-test-001",
"language": "en",
"title": "Test Advisory Title",
"summary": "Summary of advisory.",
"canonicalUrl": "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory",
"externalUrl": "https://example.com/external/advisory",
"bodyHtml": "<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>",
"bodySegments": [
"<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>"
],
"alertType": "Advisory",
"subject": "Infrastructure",
"banner": null,
"published": "2025-08-10T15:30:00Z",
"modified": "2025-08-11T12:00:00Z",
"rawCreated": "August 10, 2025",
"rawModified": "2025-08-11T12:00:00Z"
}

View File

@@ -0,0 +1,13 @@
{
"ERROR": false,
"response": [
{
"id": 396,
"title": "Advisory"
},
{
"id": 397,
"title": "Alert"
}
]
}

View File

@@ -0,0 +1,92 @@
using System;
using System.IO;
using System.Linq;
using System.Text.Json;
using FluentAssertions;
using StellaOps.Feedser.Source.Cccs.Internal;
using StellaOps.Feedser.Source.Common.Html;
using Xunit;
using Xunit.Abstractions;
namespace StellaOps.Feedser.Source.Cccs.Tests.Internal;
public sealed class CccsHtmlParserTests
{
private readonly ITestOutputHelper _output;
private static readonly HtmlContentSanitizer Sanitizer = new();
private static readonly CccsHtmlParser Parser = new(Sanitizer);
public CccsHtmlParserTests(ITestOutputHelper output)
{
_output = output ?? throw new ArgumentNullException(nameof(output));
}
public static IEnumerable<object[]> ParserCases()
{
yield return new object[]
{
"cccs-raw-advisory.json",
"TEST-001",
"en",
new[] { "Vendor Widget 1.0", "Vendor Widget 2.0" },
new[]
{
"https://example.com/details",
"https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en"
},
new[] { "CVE-2020-1234", "CVE-2021-9999" }
};
yield return new object[]
{
"cccs-raw-advisory-fr.json",
"TEST-002-FR",
"fr",
new[] { "Produit Exemple 3.1", "Produit Exemple 3.2", "Variante 3.2.1" },
new[]
{
"https://exemple.ca/details",
"https://www.cyber.gc.ca/fr/contact-centre-cyber"
},
new[] { "CVE-2024-1111" }
};
}
[Theory]
[MemberData(nameof(ParserCases))]
public void Parse_ExtractsExpectedFields(
string fixtureName,
string expectedSerial,
string expectedLanguage,
string[] expectedProducts,
string[] expectedReferenceUrls,
string[] expectedCves)
{
var raw = LoadFixture<CccsRawAdvisoryDocument>(fixtureName);
var dto = Parser.Parse(raw);
_output.WriteLine("Products: {0}", string.Join("|", dto.Products));
_output.WriteLine("References: {0}", string.Join("|", dto.References.Select(r => $"{r.Url} ({r.Label})")));
_output.WriteLine("CVEs: {0}", string.Join("|", dto.CveIds));
dto.SerialNumber.Should().Be(expectedSerial);
dto.Language.Should().Be(expectedLanguage);
dto.Products.Should().BeEquivalentTo(expectedProducts);
foreach (var url in expectedReferenceUrls)
{
dto.References.Should().Contain(reference => reference.Url == url);
}
dto.CveIds.Should().BeEquivalentTo(expectedCves);
dto.ContentHtml.Should().Contain("<ul>").And.Contain("<li>");
dto.ContentHtml.Should().Contain("<h2", because: "heading structure must survive sanitisation for UI rendering");
}
internal static T LoadFixture<T>(string fileName)
{
var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName);
var json = File.ReadAllText(path);
return JsonSerializer.Deserialize<T>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!;
}
}

View File

@@ -0,0 +1,43 @@
using System;
using FluentAssertions;
using StellaOps.Feedser.Source.Cccs.Internal;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Html;
using StellaOps.Feedser.Storage.Mongo.Documents;
using Xunit;
namespace StellaOps.Feedser.Source.Cccs.Tests.Internal;
public sealed class CccsMapperTests
{
[Fact]
public void Map_CreatesCanonicalAdvisory()
{
var raw = CccsHtmlParserTests.LoadFixture<CccsRawAdvisoryDocument>("cccs-raw-advisory.json");
var dto = new CccsHtmlParser(new HtmlContentSanitizer()).Parse(raw);
var document = new DocumentRecord(
Guid.NewGuid(),
CccsConnectorPlugin.SourceName,
dto.CanonicalUrl,
DateTimeOffset.UtcNow,
"sha-test",
DocumentStatuses.PendingMap,
"application/json",
Headers: null,
Metadata: null,
Etag: null,
LastModified: dto.Modified,
GridFsId: null);
var recordedAt = DateTimeOffset.Parse("2025-08-12T00:00:00Z");
var advisory = CccsMapper.Map(dto, document, recordedAt);
advisory.AdvisoryKey.Should().Be("TEST-001");
advisory.Title.Should().Be(dto.Title);
advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" });
advisory.References.Should().Contain(reference => reference.Url == dto.CanonicalUrl && reference.Kind == "details");
advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details");
advisory.AffectedPackages.Should().HaveCount(2);
advisory.Provenance.Should().ContainSingle(p => p.Source == CccsConnectorPlugin.SourceName && p.Kind == "advisory");
}
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
</ItemGroup>
<ItemGroup>
<None Update="Fixtures\*.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,606 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Globalization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Feedser.Source.Cccs.Configuration;
using StellaOps.Feedser.Source.Cccs.Internal;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.Cccs;
public sealed class CccsConnector : IFeedConnector
{
private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private const string DtoSchemaVersion = "cccs.dto.v1";
private readonly CccsFeedClient _feedClient;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
private readonly IDtoStore _dtoStore;
private readonly IAdvisoryStore _advisoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly CccsHtmlParser _htmlParser;
private readonly CccsDiagnostics _diagnostics;
private readonly CccsOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<CccsConnector> _logger;
public CccsConnector(
CccsFeedClient feedClient,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
CccsHtmlParser htmlParser,
CccsDiagnostics diagnostics,
IOptions<CccsOptions> options,
TimeProvider? timeProvider,
ILogger<CccsConnector> logger)
{
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_htmlParser = htmlParser ?? throw new ArgumentNullException(nameof(htmlParser));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => CccsConnectorPlugin.SourceName;
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var now = _timeProvider.GetUtcNow();
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments);
var pendingMappings = new HashSet<Guid>(cursor.PendingMappings);
var knownHashes = new Dictionary<string, string>(cursor.KnownEntryHashes, StringComparer.Ordinal);
var feedsProcessed = 0;
var totalItems = 0;
var added = 0;
var unchanged = 0;
try
{
foreach (var feed in _options.Feeds)
{
cancellationToken.ThrowIfCancellationRequested();
_diagnostics.FetchAttempt();
var result = await _feedClient.FetchAsync(feed, _options.RequestTimeout, cancellationToken).ConfigureAwait(false);
feedsProcessed++;
totalItems += result.Items.Count;
if (result.Items.Count == 0)
{
_diagnostics.FetchSuccess();
await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false);
continue;
}
var items = result.Items
.Where(static item => !string.IsNullOrWhiteSpace(item.Title))
.OrderByDescending(item => ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified) ?? DateTimeOffset.MinValue)
.ThenByDescending(item => ParseDate(item.DateCreated) ?? DateTimeOffset.MinValue)
.ToList();
foreach (var item in items)
{
cancellationToken.ThrowIfCancellationRequested();
var documentUri = BuildDocumentUri(item, feed);
var rawDocument = CreateRawDocument(item, feed, result.AlertTypes);
var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, RawSerializerOptions);
var sha = ComputeSha256(payload);
if (knownHashes.TryGetValue(documentUri, out var existingHash)
&& string.Equals(existingHash, sha, StringComparison.Ordinal))
{
unchanged++;
_diagnostics.FetchUnchanged();
continue;
}
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
if (existing is not null
&& string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)
&& string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal))
{
knownHashes[documentUri] = sha;
unchanged++;
_diagnostics.FetchUnchanged();
continue;
}
var gridFsId = await _rawDocumentStorage.UploadAsync(
SourceName,
documentUri,
payload,
"application/json",
expiresAt: null,
cancellationToken).ConfigureAwait(false);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = rawDocument.Language,
["cccs.sourceId"] = rawDocument.SourceId,
};
if (!string.IsNullOrWhiteSpace(rawDocument.SerialNumber))
{
metadata["cccs.serialNumber"] = rawDocument.SerialNumber!;
}
if (!string.IsNullOrWhiteSpace(rawDocument.AlertType))
{
metadata["cccs.alertType"] = rawDocument.AlertType!;
}
var recordId = existing?.Id ?? Guid.NewGuid();
var record = new DocumentRecord(
recordId,
SourceName,
documentUri,
now,
sha,
DocumentStatuses.PendingParse,
"application/json",
Headers: null,
Metadata: metadata,
Etag: null,
LastModified: rawDocument.Modified ?? rawDocument.Published ?? result.LastModifiedUtc,
GridFsId: gridFsId,
ExpiresAt: null);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
pendingDocuments.Add(upserted.Id);
pendingMappings.Remove(upserted.Id);
knownHashes[documentUri] = sha;
added++;
_diagnostics.FetchDocument();
if (added >= _options.MaxEntriesPerFetch)
{
break;
}
}
_diagnostics.FetchSuccess();
await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false);
if (added >= _options.MaxEntriesPerFetch)
{
break;
}
}
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException)
{
_diagnostics.FetchFailure();
_logger.LogError(ex, "CCCS fetch failed");
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
var trimmedHashes = TrimKnownHashes(knownHashes, _options.MaxKnownEntries);
var updatedCursor = cursor
.WithPendingDocuments(pendingDocuments)
.WithPendingMappings(pendingMappings)
.WithKnownEntryHashes(trimmedHashes)
.WithLastFetch(now);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"CCCS fetch completed feeds={Feeds} items={Items} newDocuments={Added} unchanged={Unchanged} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}",
feedsProcessed,
totalItems,
added,
unchanged,
pendingDocuments.Count,
pendingMappings.Count);
}
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingDocuments.Count == 0)
{
return;
}
var pendingDocuments = cursor.PendingDocuments.ToList();
var pendingMappings = cursor.PendingMappings.ToList();
var now = _timeProvider.GetUtcNow();
var parsed = 0;
var parseFailures = 0;
foreach (var documentId in cursor.PendingDocuments)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
pendingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
_diagnostics.ParseFailure();
parseFailures++;
continue;
}
if (!document.GridFsId.HasValue)
{
_diagnostics.ParseFailure();
_logger.LogWarning("CCCS document {DocumentId} missing GridFS payload", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
parseFailures++;
continue;
}
byte[] payload;
try
{
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_diagnostics.ParseFailure();
_logger.LogError(ex, "CCCS unable to download raw document {DocumentId}", documentId);
throw;
}
CccsRawAdvisoryDocument? raw;
try
{
raw = JsonSerializer.Deserialize<CccsRawAdvisoryDocument>(payload, RawSerializerOptions);
}
catch (Exception ex)
{
_diagnostics.ParseFailure();
_logger.LogWarning(ex, "CCCS failed to deserialize raw document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
parseFailures++;
continue;
}
if (raw is null)
{
_diagnostics.ParseFailure();
_logger.LogWarning("CCCS raw document {DocumentId} produced null payload", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
parseFailures++;
continue;
}
CccsAdvisoryDto dto;
try
{
dto = _htmlParser.Parse(raw);
}
catch (Exception ex)
{
_diagnostics.ParseFailure();
_logger.LogWarning(ex, "CCCS failed to parse advisory DTO for {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
parseFailures++;
continue;
}
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var dtoBson = BsonDocument.Parse(dtoJson);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
pendingDocuments.Remove(documentId);
if (!pendingMappings.Contains(documentId))
{
pendingMappings.Add(documentId);
}
_diagnostics.ParseSuccess();
parsed++;
}
var updatedCursor = cursor
.WithPendingDocuments(pendingDocuments)
.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
if (parsed > 0 || parseFailures > 0)
{
_logger.LogInformation(
"CCCS parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}",
parsed,
parseFailures,
pendingDocuments.Count,
pendingMappings.Count);
}
}
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingMappings.Count == 0)
{
return;
}
var pendingMappings = cursor.PendingMappings.ToList();
var mapped = 0;
var mappingFailures = 0;
foreach (var documentId in cursor.PendingMappings)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
pendingMappings.Remove(documentId);
_diagnostics.MapFailure();
mappingFailures++;
continue;
}
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
if (dtoRecord is null)
{
_diagnostics.MapFailure();
_logger.LogWarning("CCCS document {DocumentId} missing DTO payload", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
mappingFailures++;
continue;
}
CccsAdvisoryDto? dto;
try
{
var json = dtoRecord.Payload.ToJson();
dto = JsonSerializer.Deserialize<CccsAdvisoryDto>(json, DtoSerializerOptions);
}
catch (Exception ex)
{
_diagnostics.MapFailure();
_logger.LogWarning(ex, "CCCS failed to deserialize DTO for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
mappingFailures++;
continue;
}
if (dto is null)
{
_diagnostics.MapFailure();
_logger.LogWarning("CCCS DTO for document {DocumentId} evaluated to null", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
mappingFailures++;
continue;
}
try
{
var advisory = CccsMapper.Map(dto, document, dtoRecord.ValidatedAt);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapSuccess();
mapped++;
}
catch (Exception ex)
{
_diagnostics.MapFailure();
_logger.LogError(ex, "CCCS mapping failed for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
mappingFailures++;
}
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
if (mapped > 0 || mappingFailures > 0)
{
_logger.LogInformation(
"CCCS map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}",
mapped,
mappingFailures,
pendingMappings.Count);
}
}
private async Task<CccsCursor> GetCursorAsync(CancellationToken cancellationToken)
{
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
return state is null ? CccsCursor.Empty : CccsCursor.FromBson(state.Cursor);
}
private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}
private async Task DelayBetweenRequestsAsync(CancellationToken cancellationToken)
{
if (_options.RequestDelay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
// Ignore cancellation during delay; caller handles.
}
}
private static string BuildDocumentUri(CccsFeedItem item, CccsFeedEndpoint feed)
{
if (!string.IsNullOrWhiteSpace(item.Url))
{
if (Uri.TryCreate(item.Url, UriKind.Absolute, out var absolute))
{
return absolute.ToString();
}
var baseUri = new Uri("https://www.cyber.gc.ca", UriKind.Absolute);
if (Uri.TryCreate(baseUri, item.Url, out var combined))
{
return combined.ToString();
}
}
return $"https://www.cyber.gc.ca/api/cccs/threats/{feed.Language}/{item.Nid}";
}
private static CccsRawAdvisoryDocument CreateRawDocument(CccsFeedItem item, CccsFeedEndpoint feed, IReadOnlyDictionary<int, string> taxonomy)
{
var language = string.IsNullOrWhiteSpace(item.Language) ? feed.Language : item.Language!.Trim();
var identifier = !string.IsNullOrWhiteSpace(item.SerialNumber)
? item.SerialNumber!.Trim()
: !string.IsNullOrWhiteSpace(item.Uuid)
? item.Uuid!.Trim()
: $"nid-{item.Nid}";
var canonicalUrl = BuildDocumentUri(item, feed);
var bodySegments = item.Body ?? Array.Empty<string>();
var bodyHtml = string.Join(Environment.NewLine, bodySegments);
var published = ParseDate(item.DateCreated);
var modified = ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified);
var alertType = ResolveAlertType(item, taxonomy);
return new CccsRawAdvisoryDocument
{
SourceId = identifier,
SerialNumber = item.SerialNumber?.Trim(),
Uuid = item.Uuid,
Language = language.ToLowerInvariant(),
Title = item.Title?.Trim() ?? identifier,
Summary = item.Summary?.Trim(),
CanonicalUrl = canonicalUrl,
ExternalUrl = item.ExternalUrl,
BodyHtml = bodyHtml,
BodySegments = bodySegments,
AlertType = alertType,
Subject = item.Subject,
Banner = item.Banner,
Published = published,
Modified = modified,
RawDateCreated = item.DateCreated,
RawDateModified = item.DateModifiedTimestamp ?? item.DateModified,
};
}
private static string? ResolveAlertType(CccsFeedItem item, IReadOnlyDictionary<int, string> taxonomy)
{
if (item.AlertType.ValueKind == JsonValueKind.Number)
{
var id = item.AlertType.GetInt32();
return taxonomy.TryGetValue(id, out var label) ? label : id.ToString(CultureInfo.InvariantCulture);
}
if (item.AlertType.ValueKind == JsonValueKind.String)
{
return item.AlertType.GetString();
}
if (item.AlertType.ValueKind == JsonValueKind.Array)
{
foreach (var element in item.AlertType.EnumerateArray())
{
if (element.ValueKind == JsonValueKind.Number)
{
var id = element.GetInt32();
if (taxonomy.TryGetValue(id, out var label))
{
return label;
}
}
else if (element.ValueKind == JsonValueKind.String)
{
var label = element.GetString();
if (!string.IsNullOrWhiteSpace(label))
{
return label;
}
}
}
}
return null;
}
private static Dictionary<string, string> TrimKnownHashes(Dictionary<string, string> hashes, int maxEntries)
{
if (hashes.Count <= maxEntries)
{
return hashes;
}
var overflow = hashes.Count - maxEntries;
foreach (var key in hashes.Keys.Take(overflow).ToList())
{
hashes.Remove(key);
}
return hashes;
}
private static DateTimeOffset? ParseDate(string? value)
=> string.IsNullOrWhiteSpace(value)
? null
: DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
? parsed
: null;
private static string ComputeSha256(byte[] payload)
=> Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
}

View File

@@ -0,0 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.Cccs;
public sealed class CccsConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cccs";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CccsConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CccsConnector>();
}
}

View File

@@ -0,0 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Feedser.Core.Jobs;
using StellaOps.Feedser.Source.Cccs.Configuration;
namespace StellaOps.Feedser.Source.Cccs;
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "feedser:sources:cccs";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCccsConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CccsFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -0,0 +1,47 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.Cccs.Configuration;
using StellaOps.Feedser.Source.Cccs.Internal;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Common.Html;
namespace StellaOps.Feedser.Source.Cccs;
public static class CccsServiceCollectionExtensions
{
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CccsOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
clientOptions.UserAgent = "StellaOps.Feedser.Cccs/1.0";
clientOptions.Timeout = options.RequestTimeout;
clientOptions.AllowedHosts.Clear();
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
{
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
}
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
clientOptions.AllowedHosts.Add("cyber.gc.ca");
});
services.TryAddSingleton<HtmlContentSanitizer>();
services.TryAddSingleton<CccsDiagnostics>();
services.TryAddSingleton<CccsHtmlParser>();
services.TryAddSingleton<CccsFeedClient>();
services.AddTransient<CccsConnector>();
return services;
}
}

View File

@@ -1,29 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.Cccs;
public sealed class CccsConnectorPlugin : IConnectorPlugin
{
public string Name => "cccs";
public bool IsAvailable(IServiceProvider services) => true;
public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name);
private sealed class StubConnector : IFeedConnector
{
public StubConnector(string sourceName) => SourceName = sourceName;
public string SourceName { get; }
public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
}
}

View File

@@ -0,0 +1,175 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Feedser.Source.Cccs.Configuration;
public sealed class CccsOptions
{
public const string HttpClientName = "feedser.source.cccs";
private readonly List<CccsFeedEndpoint> _feeds = new();
public CccsOptions()
{
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
}
/// <summary>
/// Feed endpoints to poll; configure per language or content category.
/// </summary>
public IList<CccsFeedEndpoint> Feeds => _feeds;
/// <summary>
/// Maximum number of entries to enqueue per fetch cycle.
/// </summary>
public int MaxEntriesPerFetch { get; set; } = 80;
/// <summary>
/// Maximum remembered entries (URI+hash) for deduplication.
/// </summary>
public int MaxKnownEntries { get; set; } = 512;
/// <summary>
/// Timeout applied to feed and taxonomy requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay between successive feed requests to respect upstream throttling.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when fetch fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
public void Validate()
{
if (_feeds.Count == 0)
{
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
}
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var feed in _feeds)
{
feed.Validate();
if (!seenLanguages.Add(feed.Language))
{
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
}
}
if (MaxEntriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
}
if (MaxKnownEntries <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}
public sealed class CccsFeedEndpoint
{
public CccsFeedEndpoint()
{
}
public CccsFeedEndpoint(string language, Uri uri)
{
Language = language;
Uri = uri;
}
public string Language { get; set; } = "en";
public Uri? Uri { get; set; }
public void Validate()
{
if (string.IsNullOrWhiteSpace(Language))
{
throw new InvalidOperationException("CCCS feed language must be specified.");
}
if (Uri is null || !Uri.IsAbsoluteUri)
{
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
}
}
public Uri BuildTaxonomyUri()
{
if (Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
}
var language = Uri.GetQueryParameterValueOrDefault("lang", Language);
var builder = $"https://www.cyber.gc.ca/api/cccs/taxonomy/v1/get?lang={language}&vocabulary=cccs_alert_type";
return new Uri(builder, UriKind.Absolute);
}
}
internal static class CccsUriExtensions
{
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
{
if (uri is null)
{
return fallback;
}
var query = uri.Query;
if (string.IsNullOrEmpty(query))
{
return fallback;
}
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex < 0)
{
continue;
}
var left = pair[..separatorIndex].Trim();
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var right = pair[(separatorIndex + 1)..].Trim();
if (right.Length == 0)
{
continue;
}
return Uri.UnescapeDataString(right);
}
return fallback;
}
}

View File

@@ -0,0 +1,54 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.Cccs.Internal;
internal sealed record CccsAdvisoryDto
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string SerialNumber { get; init; } = string.Empty;
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("products")]
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
[JsonPropertyName("references")]
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
}
internal sealed record CccsReferenceDto(
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("label")] string? Label);

View File

@@ -0,0 +1,145 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Feedser.Source.Cccs.Internal;
internal sealed record CccsCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyDictionary<string, string> KnownEntryHashes,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
return this with { PendingMappings = distinct };
}
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
{
var map = hashes is null || hashes.Count == 0
? EmptyHashes
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
return this with { KnownEntryHashes = map };
}
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var doc = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (KnownEntryHashes.Count > 0)
{
var hashes = new BsonArray();
foreach (var kvp in KnownEntryHashes)
{
hashes.Add(new BsonDocument
{
["uri"] = kvp.Key,
["hash"] = kvp.Value,
});
}
doc["knownEntryHashes"] = hashes;
}
if (LastFetchAt.HasValue)
{
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return doc;
}
public static CccsCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var hashes = ReadHashMap(document);
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
? ParseDateTime(value)
: null;
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidCollection;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
items.Add(guid);
}
}
return items;
}
private static IReadOnlyDictionary<string, string> ReadHashMap(BsonDocument document)
{
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0)
{
return EmptyHashes;
}
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
foreach (var element in array)
{
if (element is not BsonDocument entry)
{
continue;
}
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString))
{
continue;
}
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull
? hashValue.AsString
: string.Empty;
map[uriValue.AsString] = hash;
}
return map;
}
private static DateTimeOffset? ParseDateTime(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,58 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Feedser.Source.Cccs.Internal;
public sealed class CccsDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Feedser.Source.Cccs";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CccsDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchSuccess() => _fetchSuccess.Add(1);
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void ParseQuarantine() => _parseQuarantine.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,146 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Feedser.Source.Cccs.Configuration;
using StellaOps.Feedser.Source.Common.Fetch;
namespace StellaOps.Feedser.Source.Cccs.Internal;
public sealed class CccsFeedClient
{
private static readonly string[] AcceptHeaders =
{
"application/json",
"application/vnd.api+json;q=0.9",
"text/json;q=0.8",
"application/*+json;q=0.7",
};
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(endpoint);
if (endpoint.Uri is null)
{
throw new InvalidOperationException("Feed endpoint URI must be configured.");
}
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = requestTimeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.feedUri"] = endpoint.Uri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
return CccsFeedResult.Empty;
}
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
if (feedResponse is null || feedResponse.Error)
{
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
return CccsFeedResult.Empty;
}
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
return new CccsFeedResult(items, taxonomy, result.LastModified);
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
throw;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
throw;
}
}
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
{
var taxonomyUri = endpoint.BuildTaxonomyUri();
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
{
AcceptHeaders = AcceptHeaders,
TimeoutOverride = timeout,
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["cccs.language"] = endpoint.Language,
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
},
};
try
{
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
foreach (var item in taxonomyResponse.Response)
{
if (!string.IsNullOrWhiteSpace(item.Title))
{
map[item.Id] = item.Title!;
}
}
return map;
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
return new Dictionary<int, string>(0);
}
}
private static T? Deserialize<T>(byte[] content)
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
}

View File

@@ -0,0 +1,101 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.Cccs.Internal;
internal sealed class CccsFeedResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsFeedItem> Response { get; init; } = new();
}
internal sealed class CccsFeedItem
{
[JsonPropertyName("nid")]
public int Nid { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("lang")]
public string? Language { get; init; }
[JsonPropertyName("date_modified")]
public string? DateModified { get; init; }
[JsonPropertyName("date_modified_ts")]
public string? DateModifiedTimestamp { get; init; }
[JsonPropertyName("date_created")]
public string? DateCreated { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("body")]
public string[] Body { get; init; } = Array.Empty<string>();
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("alert_type")]
public JsonElement AlertType { get; init; }
[JsonPropertyName("serial_number")]
public string? SerialNumber { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("moderation_state")]
public string? ModerationState { get; init; }
[JsonPropertyName("external_url")]
public string? ExternalUrl { get; init; }
}
internal sealed class CccsTaxonomyResponse
{
[JsonPropertyName("ERROR")]
public bool Error { get; init; }
[JsonPropertyName("response")]
public List<CccsTaxonomyItem> Response { get; init; } = new();
}
internal sealed class CccsTaxonomyItem
{
[JsonPropertyName("id")]
public int Id { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
}
internal sealed record CccsFeedResult(
IReadOnlyList<CccsFeedItem> Items,
IReadOnlyDictionary<int, string> AlertTypes,
DateTimeOffset? LastModifiedUtc)
{
public static CccsFeedResult Empty { get; } = new(
Array.Empty<CccsFeedItem>(),
new Dictionary<int, string>(0),
null);
}
internal static class CccsFeedResultExtensions
{
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
=> new(items, alertTypes, lastModified);
}

View File

@@ -0,0 +1,449 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using StellaOps.Feedser.Source.Common.Html;
namespace StellaOps.Feedser.Source.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
private static readonly CultureInfo[] EnglishCultures =
{
CultureInfo.GetCultureInfo("en-CA"),
CultureInfo.GetCultureInfo("en-US"),
CultureInfo.InvariantCulture,
};
private static readonly CultureInfo[] FrenchCultures =
{
CultureInfo.GetCultureInfo("fr-CA"),
CultureInfo.GetCultureInfo("fr-FR"),
CultureInfo.InvariantCulture,
};
private static readonly string[] ProductHeadingKeywords =
{
"affected",
"produit",
"produits",
"produits touch",
"produits concern",
"mesures recommand",
};
private static readonly string[] TrackingParameterPrefixes =
{
"utm_",
"mc_",
"mkt_",
"elq",
};
private readonly HtmlContentSanitizer _sanitizer;
private readonly HtmlParser _parser;
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
{
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
_parser = new HtmlParser(new HtmlParserOptions
{
IsScripting = false,
IsKeepingSourceReferences = false,
});
}
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
{
ArgumentNullException.ThrowIfNull(raw);
var baseUri = TryCreateUri(raw.CanonicalUrl);
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
var body = document.Body ?? document.DocumentElement;
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
var contentRoot = body ?? document.DocumentElement;
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
? raw.SerialNumber!.Trim()
: ExtractSerialNumber(document) ?? raw.SourceId;
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
var products = ExtractProducts(contentRoot);
var cveIds = ExtractCveIds(document);
return new CccsAdvisoryDto
{
SourceId = raw.SourceId,
SerialNumber = serialNumber,
Language = raw.Language,
Title = raw.Title,
Summary = CollapseWhitespace(raw.Summary),
CanonicalUrl = raw.CanonicalUrl,
ContentHtml = sanitized,
Published = published,
Modified = raw.Modified ?? published,
AlertType = raw.AlertType,
Subject = raw.Subject,
Products = products,
References = references,
CveIds = cveIds,
};
}
private static Uri? TryCreateUri(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
}
private static string? ExtractSerialNumber(IDocument document)
{
if (document.Body is null)
{
return null;
}
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = SerialRegex.Match(text);
if (match.Success && match.Groups["id"].Success)
{
var value = match.Groups["id"].Value.Trim();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
var bodyText = document.Body.TextContent;
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
return fallback.Success && fallback.Groups["id"].Success
? fallback.Groups["id"].Value.Trim()
: null;
}
private static DateTimeOffset? ExtractDate(IDocument document, string language)
{
if (document.Body is null)
{
return null;
}
var textSegments = new List<string>();
foreach (var element in document.QuerySelectorAll("strong, p, div"))
{
var text = element.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
var match = DateRegex.Match(text);
if (match.Success && match.Groups["date"].Success)
{
textSegments.Add(match.Groups["date"].Value.Trim());
}
}
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
{
textSegments.Add(document.Body.TextContent);
}
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
foreach (var segment in textSegments)
{
foreach (var culture in cultures)
{
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return new DateTimeOffset(parsed.ToUniversalTime());
}
}
}
return null;
}
private static IReadOnlyList<string> ExtractProducts(IElement? root)
{
if (root is null)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
{
var text = heading.TextContent?.Trim();
if (!IsProductHeading(text))
{
continue;
}
var sibling = heading.NextElementSibling;
while (sibling is not null)
{
if (IsHeading(sibling))
{
break;
}
if (IsListElement(sibling))
{
AppendListItems(sibling, results);
if (results.Count > 0)
{
break;
}
}
else if (IsContentContainer(sibling))
{
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
{
AppendListItems(list, results);
}
if (results.Count > 0)
{
break;
}
}
sibling = sibling.NextElementSibling;
}
if (results.Count > 0)
{
break;
}
}
if (results.Count == 0)
{
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
{
var itemText = CollapseWhitespace(li.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
results.Add(itemText);
}
}
}
return results.Count == 0
? Array.Empty<string>()
: results
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static bool IsProductHeading(string? heading)
{
if (string.IsNullOrWhiteSpace(heading))
{
return false;
}
var lowered = heading.ToLowerInvariant();
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
}
private static bool IsHeading(IElement element)
=> element.LocalName.Length == 2
&& element.LocalName[0] == 'h'
&& char.IsDigit(element.LocalName[1]);
private static bool IsListElement(IElement element)
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
private static bool IsContentContainer(IElement element)
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
{
foreach (var li in listElement.QuerySelectorAll("li"))
{
if (li is null)
{
continue;
}
var clone = li.Clone(true) as IElement;
if (clone is null)
{
continue;
}
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
{
nested.Remove();
}
var itemText = CollapseWhitespace(clone.TextContent);
if (!string.IsNullOrWhiteSpace(itemText))
{
buffer.Add(itemText);
}
}
}
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
{
if (root is null)
{
return Array.Empty<CccsReferenceDto>();
}
var references = new List<CccsReferenceDto>();
foreach (var anchor in root.QuerySelectorAll("a[href]"))
{
var href = anchor.GetAttribute("href");
var normalized = NormalizeReferenceUrl(href, baseUri, language);
if (normalized is null)
{
continue;
}
var label = CollapseWhitespace(anchor.TextContent);
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
}
return references.Count == 0
? Array.Empty<CccsReferenceDto>()
: references
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
{
if (string.IsNullOrWhiteSpace(href))
{
return null;
}
if (!Uri.TryCreate(href, UriKind.Absolute, out var absolute))
{
if (baseUri is null || !Uri.TryCreate(baseUri, href, out absolute))
{
return null;
}
}
var builder = new UriBuilder(absolute)
{
Fragment = string.Empty,
};
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
builder.Query = filteredQuery;
return builder.Uri.ToString();
}
private static string FilterTrackingParameters(string query, Uri uri, string language)
{
if (string.IsNullOrWhiteSpace(query))
{
return string.Empty;
}
var trimmed = query.TrimStart('?');
if (string.IsNullOrWhiteSpace(trimmed))
{
return string.Empty;
}
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
var kept = new List<string>();
foreach (var parameter in parameters)
{
var separatorIndex = parameter.IndexOf('=');
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
{
kept.Add($"lang={language}");
continue;
}
kept.Add(parameter);
}
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
{
kept.Add($"lang={language}");
}
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
}
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
{
if (document.Body is null)
{
return Array.Empty<string>();
}
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
if (matches.Count == 0)
{
return Array.Empty<string>();
}
return matches
.Select(match => match.Value.ToUpperInvariant())
.Distinct(StringComparer.Ordinal)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
}
private static string? CollapseWhitespace(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
return collapsed.Length == 0 ? null : collapsed;
}
}

View File

@@ -0,0 +1,151 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Storage.Mongo.Documents;
namespace StellaOps.Feedser.Source.Cccs.Internal;
internal static class CccsMapper
{
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"advisory",
dto.AlertType ?? dto.SerialNumber,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory })
};
return new Advisory(
advisoryKey: dto.SerialNumber,
title: dto.Title,
summary: dto.Summary,
language: dto.Language,
published: dto.Published ?? dto.Modified,
modified: dto.Modified ?? dto.Published,
severity: null,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: provenance);
}
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4)
{
dto.SerialNumber,
};
if (!string.IsNullOrWhiteSpace(dto.SourceId)
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
{
aliases.Add(dto.SourceId);
}
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
dto.CanonicalUrl,
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
"reference",
"cccs",
reference.Label,
new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
if (string.IsNullOrWhiteSpace(product))
{
continue;
}
var identifier = product.Trim();
var provenance = new AdvisoryProvenance(
CccsConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: Array.Empty<AffectedVersionRange>(),
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
}

View File

@@ -0,0 +1,58 @@
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.Cccs.Internal;
internal sealed record CccsRawAdvisoryDocument
{
[JsonPropertyName("sourceId")]
public string SourceId { get; init; } = string.Empty;
[JsonPropertyName("serialNumber")]
public string? SerialNumber { get; init; }
[JsonPropertyName("uuid")]
public string? Uuid { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "en";
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("canonicalUrl")]
public string CanonicalUrl { get; init; } = string.Empty;
[JsonPropertyName("externalUrl")]
public string? ExternalUrl { get; init; }
[JsonPropertyName("bodyHtml")]
public string BodyHtml { get; init; } = string.Empty;
[JsonPropertyName("bodySegments")]
public string[] BodySegments { get; init; } = Array.Empty<string>();
[JsonPropertyName("alertType")]
public string? AlertType { get; init; }
[JsonPropertyName("subject")]
public string? Subject { get; init; }
[JsonPropertyName("banner")]
public string? Banner { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("rawCreated")]
public string? RawDateCreated { get; init; }
[JsonPropertyName("rawModified")]
public string? RawDateModified { get; init; }
}

View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Feedser.Core.Jobs;
namespace StellaOps.Feedser.Source.Cccs;
internal static class CccsJobKinds
{
public const string Fetch = "source:cccs:fetch";
}
internal sealed class CccsFetchJob : IJob
{
private readonly CccsConnector _connector;
public CccsFetchJob(CccsConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Cccs.Tests")]

View File

@@ -6,11 +6,11 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
</ItemGroup>
</Project>
<ItemGroup>
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
</ItemGroup>
</Project>

View File

@@ -2,9 +2,10 @@
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|FEEDCONN-CCCS-02-001 Catalogue official CCCS advisory feeds|BE-Conn-CCCS|Research|**DONE (2025-10-11)** Resolved RSS→Atom redirects (`/api/cccs/rss/v1/get?...``/api/cccs/atom/v1/get?...`), confirmed feed caps at 50 entries with inline HTML bodies, no `Last-Modified`/`ETag`, and `updated` timestamps in UTC. Findings and packet captures parked in `docs/feedser-connector-research-20251011.md`; retention sweep follow-up tracked in 02-007.|
|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**TODO** Register HTTP client with redirect allowance + `User-Agent` override, persist Atom payload plus derived SHA256 to `document` store, and use feed-level `<updated>` / entry `<updated>` for cursoring. Capture absence of cache headers by stamping synthetic `fetchedAt` TTL and throttling retries to 1/min (Azure App Gateway fronted).|
|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**TODO** Build DTO that keeps original HTML in `ContentHtml` and extracts structured fields: `Serial number`, `Date`, product bullet lists, and reference hyperlinks. Strip tracking query params, collapse whitespace, and normalise French vs English feeds via `lang=` query.|
|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**TODO** Map advisories into canonical records with aliases, references, vendor/package range primitives, and provenance. Align normalized SemVer rules per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.<br>2025-10-11 research trail: emit `NormalizedVersions` like `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"cccs:bulletin-id"}]`; include provenance notes to keep storage decision reasons.|
|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** Add regression tests with canned fixtures; support `UPDATE_CCCS_FIXTURES=1` to refresh snapshots.|
|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**TODO** Document connector configuration, add logging/metrics, and update backlog once feature-complete.|
|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**TODO** HTML index exposes `?page=<n>` pagination (`https://www.cyber.gc.ca/en/alerts-advisories?page=4` tested OK). Need to measure depth, record earliest advisory date, and confirm whether separate feeds exist for bulletins/vulnerabilities. Produce backfill plan (HTML scrape → Atom transformation) and language split guidance.|
|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**DONE (2025-10-14)** `CccsConnector.FetchAsync` now hydrates feeds via `CccsFeedClient`, persists per-entry JSON payloads with SHA256 dedupe and cursor state, throttles requests, and records taxonomy + language metadata in document state.|
|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**DONE (2025-10-14)** Added `CccsHtmlParser` to sanitize Atom body HTML, extract serial/date/product bullets, collapse whitespace, and emit normalized reference URLs; `ParseAsync` now persists DTO records under schema `cccs.dto.v1`.|
|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**DONE (2025-10-14)** `CccsMapper` now materializes canonical advisories (aliases from serial/source/CVEs, references incl. canonical URL, vendor package records) with provenance masks; `MapAsync` stores results in `AdvisoryStore`.|
|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** Added English/French fixtures plus parser + connector end-to-end tests (`StellaOps.Feedser.Source.Cccs.Tests`). Canned HTTP handler + Mongo fixture enables fetch→parse→map regression; fixtures refresh via `UPDATE_CCCS_FIXTURES=1`.|
|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**DONE (2025-10-15)** Added `CccsDiagnostics` meter (fetch/parse/map counters), enriched connector logs with document counts, and published `docs/ops/feedser-cccs-operations.md` covering config, telemetry, and sanitiser guidance.|
|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**DONE (2025-10-15)** Measured `/api/cccs/threats/v1/get` inventory (~5.1k rows/lang; earliest 2018-06-08), documented backfill workflow + language split strategy, and linked the runbook for Offline Kit execution.|
|FEEDCONN-CCCS-02-008 Raw DOM parsing refinement|BE-Conn-CCCS|Source.Common|**DONE (2025-10-15)** Parser now walks unsanitised DOM (heading + nested list coverage), sanitizer keeps `<h#>`/`section` nodes, and regression fixtures/tests assert EN/FR list handling + preserved HTML structure.|

View File

@@ -0,0 +1,188 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Feedser.Source.CertBund.Configuration;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Source.Common.Testing;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Feedser.Testing;
using Xunit;
namespace StellaOps.Feedser.Source.CertBund.Tests;
[Collection("mongo-fixture")]
public sealed class CertBundConnectorTests : IAsyncLifetime
{
private static readonly Uri FeedUri = new("https://test.local/content/public/securityAdvisory/rss");
private static readonly Uri PortalUri = new("https://test.local/portal/");
private static readonly Uri DetailUri = new("https://test.local/portal/api/securityadvisory?name=WID-SEC-2025-2264");
private readonly MongoIntegrationFixture _fixture;
private readonly CannedHttpMessageHandler _handler;
public CertBundConnectorTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
_handler = new CannedHttpMessageHandler();
}
[Fact]
public async Task FetchParseMap_ProducesCanonicalAdvisory()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses();
var connector = provider.GetRequiredService<CertBundConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
advisories.Should().HaveCount(1);
var advisory = advisories[0];
advisory.AdvisoryKey.Should().Be("WID-SEC-2025-2264");
advisory.Aliases.Should().Contain("CVE-2025-1234");
advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("Ivanti"));
advisory.References.Should().Contain(reference => reference.Url == DetailUri.ToString());
advisory.Language.Should().Be("de");
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None);
state.Should().NotBeNull();
state!.Cursor.Should().NotBeNull();
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
pendingDocs!.AsBsonArray.Should().BeEmpty();
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
pendingMappings!.AsBsonArray.Should().BeEmpty();
}
[Fact]
public async Task Fetch_PersistsDocumentWithMetadata()
{
await using var provider = await BuildServiceProviderAsync();
SeedResponses();
var connector = provider.GetRequiredService<CertBundConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var document = await documentStore.FindBySourceAndUriAsync(CertBundConnectorPlugin.SourceName, DetailUri.ToString(), CancellationToken.None);
document.Should().NotBeNull();
document!.Metadata.Should().ContainKey("certbund.advisoryId").WhoseValue.Should().Be("WID-SEC-2025-2264");
document.Metadata.Should().ContainKey("certbund.category");
document.Metadata.Should().ContainKey("certbund.published");
document.Status.Should().Be(DocumentStatuses.PendingParse);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None);
state.Should().NotBeNull();
state!.Cursor.Should().NotBeNull();
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
pendingDocs!.AsBsonArray.Should().HaveCount(1);
}
private async Task<ServiceProvider> BuildServiceProviderAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddCertBundConnector(options =>
{
options.FeedUri = FeedUri;
options.PortalBootstrapUri = PortalUri;
options.DetailApiUri = new Uri("https://test.local/portal/api/securityadvisory");
options.RequestDelay = TimeSpan.Zero;
options.MaxAdvisoriesPerFetch = 10;
options.MaxKnownAdvisories = 32;
});
services.Configure<HttpClientFactoryOptions>(CertBundOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
{
builder.PrimaryHandler = _handler;
});
});
var provider = services.BuildServiceProvider();
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
return provider;
}
private void SeedResponses()
{
AddJsonResponse(DetailUri, ReadFixture("certbund-detail.json"));
AddXmlResponse(FeedUri, ReadFixture("certbund-feed.xml"), "application/rss+xml");
AddHtmlResponse(PortalUri, "<html><body>OK</body></html>");
}
private void AddJsonResponse(Uri uri, string json, string? etag = null)
{
_handler.AddResponse(uri, () =>
{
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(json, Encoding.UTF8, "application/json"),
};
if (!string.IsNullOrWhiteSpace(etag))
{
response.Headers.ETag = new EntityTagHeaderValue(etag);
}
return response;
});
}
private void AddXmlResponse(Uri uri, string xml, string contentType)
{
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(xml, Encoding.UTF8, contentType),
});
}
private void AddHtmlResponse(Uri uri, string html)
{
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(html, Encoding.UTF8, "text/html"),
});
}
private static string ReadFixture(string fileName)
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => Task.CompletedTask;
}

View File

@@ -0,0 +1,36 @@
{
"name": "WID-SEC-2025-2264",
"title": "Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung",
"summary": "Ein entfernter, anonymer Angreifer kann mehrere Schwachstellen in Ivanti Endpoint Manager ausnutzen.",
"description": "<p>Ivanti Endpoint Manager weist mehrere Schwachstellen auf.</p><p>Ein Angreifer kann beliebigen Code ausführen.</p>",
"severity": "hoch",
"language": "de",
"published": "2025-10-14T06:24:49Z",
"updated": "2025-10-14T07:00:00Z",
"cveIds": [
"CVE-2025-1234",
"CVE-2025-5678"
],
"references": [
{
"url": "https://example.com/vendor/advisory",
"label": "Vendor Advisory"
},
{
"url": "https://example.com/mitre",
"label": "MITRE"
}
],
"products": [
{
"vendor": "Ivanti",
"name": "Endpoint Manager",
"versions": "2023.1 bis 2024.2"
},
{
"vendor": "Ivanti",
"name": "Endpoint Manager Cloud",
"versions": "alle"
}
]
}

View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>BSI Warn- und Informationsdienst</title>
<link>https://wid.cert-bund.de/portal/wid/securityadvisory</link>
<description>Test feed</description>
<pubDate>Tue, 14 Oct 2025 07:06:21 GMT</pubDate>
<item>
<title>[hoch] Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung</title>
<link>https://wid.cert-bund.de/portal/wid/securityadvisory?name=WID-SEC-2025-2264</link>
<category>hoch</category>
<pubDate>Tue, 14 Oct 2025 06:24:49 GMT</pubDate>
</item>
</channel>
</rss>

View File

@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
</ItemGroup>
<ItemGroup>
<None Update="Fixtures\*.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Fixtures\*.xml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,435 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Feedser.Source.CertBund.Configuration;
using StellaOps.Feedser.Source.CertBund.Internal;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Source.Common.Html;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.CertBund;
public sealed class CertBundConnector : IFeedConnector
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
};
private readonly CertBundFeedClient _feedClient;
private readonly CertBundDetailParser _detailParser;
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
private readonly IDtoStore _dtoStore;
private readonly IAdvisoryStore _advisoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly CertBundOptions _options;
private readonly TimeProvider _timeProvider;
private readonly CertBundDiagnostics _diagnostics;
private readonly ILogger<CertBundConnector> _logger;
public CertBundConnector(
CertBundFeedClient feedClient,
CertBundDetailParser detailParser,
SourceFetchService fetchService,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
IOptions<CertBundOptions> options,
CertBundDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<CertBundConnector> logger)
{
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
_detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser));
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => CertBundConnectorPlugin.SourceName;
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
IReadOnlyList<CertBundFeedItem> feedItems;
_diagnostics.FeedFetchAttempt();
try
{
feedItems = await _feedClient.LoadAsync(cancellationToken).ConfigureAwait(false);
_diagnostics.FeedFetchSuccess(feedItems.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-Bund feed fetch failed");
_diagnostics.FeedFetchFailure();
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
var coverageDays = CalculateCoverageDays(feedItems, now);
_diagnostics.RecordFeedCoverage(coverageDays);
if (feedItems.Count == 0)
{
await UpdateCursorAsync(cursor.WithLastFetch(now), cancellationToken).ConfigureAwait(false);
return;
}
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
var pendingMappings = cursor.PendingMappings.ToHashSet();
var knownAdvisories = new HashSet<string>(cursor.KnownAdvisories, StringComparer.OrdinalIgnoreCase);
var processed = 0;
var alreadyKnown = 0;
var notModified = 0;
var detailFailures = 0;
var truncated = false;
var latestPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
foreach (var item in feedItems.OrderByDescending(static i => i.Published))
{
cancellationToken.ThrowIfCancellationRequested();
if (knownAdvisories.Contains(item.AdvisoryId))
{
alreadyKnown++;
continue;
}
if (processed >= _options.MaxAdvisoriesPerFetch)
{
truncated = true;
break;
}
try
{
_diagnostics.DetailFetchAttempt();
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailUri.ToString(), cancellationToken).ConfigureAwait(false);
var request = new SourceFetchRequest(CertBundOptions.HttpClientName, SourceName, item.DetailUri)
{
AcceptHeaders = new[] { "application/json", "text/json" },
Metadata = CertBundDocumentMetadata.CreateMetadata(item),
ETag = existing?.Etag,
LastModified = existing?.LastModified,
TimeoutOverride = _options.RequestTimeout,
};
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
if (result.IsNotModified)
{
_diagnostics.DetailFetchNotModified();
notModified++;
knownAdvisories.Add(item.AdvisoryId);
continue;
}
if (!result.IsSuccess || result.Document is null)
{
_diagnostics.DetailFetchFailure("skipped");
detailFailures++;
continue;
}
_diagnostics.DetailFetchSuccess();
pendingDocuments.Add(result.Document.Id);
pendingMappings.Remove(result.Document.Id);
knownAdvisories.Add(item.AdvisoryId);
processed++;
if (_options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-Bund detail fetch failed for {AdvisoryId}", item.AdvisoryId);
_diagnostics.DetailFetchFailure("exception");
detailFailures++;
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
if (item.Published > latestPublished)
{
latestPublished = item.Published;
}
}
_diagnostics.DetailFetchEnqueued(processed);
if (feedItems.Count > 0 || processed > 0 || detailFailures > 0)
{
_logger.LogInformation(
"CERT-Bund fetch cycle: feed items {FeedItems}, enqueued {Enqueued}, already known {Known}, not modified {NotModified}, detail failures {DetailFailures}, pending documents {PendingDocuments}, pending mappings {PendingMappings}, truncated {Truncated}, coverageDays={CoverageDays}",
feedItems.Count,
processed,
alreadyKnown,
notModified,
detailFailures,
pendingDocuments.Count,
pendingMappings.Count,
truncated,
coverageDays ?? double.NaN);
}
var trimmedKnown = knownAdvisories.Count > _options.MaxKnownAdvisories
? knownAdvisories.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase)
.Take(_options.MaxKnownAdvisories)
.ToArray()
: knownAdvisories.ToArray();
var updatedCursor = cursor
.WithPendingDocuments(pendingDocuments)
.WithPendingMappings(pendingMappings)
.WithKnownAdvisories(trimmedKnown)
.WithLastPublished(latestPublished == DateTimeOffset.MinValue ? cursor.LastPublished : latestPublished)
.WithLastFetch(now);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingDocuments.Count == 0)
{
return;
}
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
var pendingMappings = cursor.PendingMappings.ToHashSet();
var now = _timeProvider.GetUtcNow();
var parsedCount = 0;
var failedCount = 0;
foreach (var documentId in cursor.PendingDocuments)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
if (!document.GridFsId.HasValue)
{
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
_diagnostics.ParseFailure("missing_payload");
failedCount++;
continue;
}
byte[] payload;
try
{
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-Bund unable to download document {DocumentId}", document.Id);
_diagnostics.ParseFailure("download_failed");
throw;
}
CertBundAdvisoryDto dto;
try
{
dto = _detailParser.Parse(new Uri(document.Uri), new Uri(document.Metadata?["certbund.portalUri"] ?? document.Uri), payload);
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-Bund failed to parse advisory detail {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
_diagnostics.ParseFailure("parse_error");
failedCount++;
continue;
}
_diagnostics.ParseSuccess(dto.Products.Count, dto.CveIds.Count);
parsedCount++;
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", bson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Add(document.Id);
}
if (cursor.PendingDocuments.Count > 0)
{
_logger.LogInformation(
"CERT-Bund parse cycle: parsed {Parsed}, failures {Failures}, remaining documents {RemainingDocuments}, pending mappings {PendingMappings}",
parsedCount,
failedCount,
remainingDocuments.Count,
pendingMappings.Count);
}
var updatedCursor = cursor
.WithPendingDocuments(remainingDocuments)
.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingMappings.Count == 0)
{
return;
}
var pendingMappings = cursor.PendingMappings.ToHashSet();
var mappedCount = 0;
var failedCount = 0;
foreach (var documentId in cursor.PendingMappings)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
pendingMappings.Remove(documentId);
continue;
}
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
if (dtoRecord is null)
{
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapFailure("missing_dto");
failedCount++;
continue;
}
CertBundAdvisoryDto? dto;
try
{
dto = JsonSerializer.Deserialize<CertBundAdvisoryDto>(dtoRecord.Payload.ToJson(), SerializerOptions);
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-Bund failed to deserialize DTO for document {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapFailure("deserialize_failed");
failedCount++;
continue;
}
if (dto is null)
{
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapFailure("null_dto");
failedCount++;
continue;
}
try
{
var advisory = CertBundMapper.Map(dto, document, dtoRecord.ValidatedAt);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapSuccess(advisory.AffectedPackages.Length, advisory.Aliases.Length);
mappedCount++;
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-Bund mapping failed for document {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapFailure("exception");
failedCount++;
}
}
if (cursor.PendingMappings.Count > 0)
{
_logger.LogInformation(
"CERT-Bund map cycle: mapped {Mapped}, failures {Failures}, remaining pending mappings {PendingMappings}",
mappedCount,
failedCount,
pendingMappings.Count);
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
private static double? CalculateCoverageDays(IReadOnlyList<CertBundFeedItem> items, DateTimeOffset fetchedAt)
{
if (items is null || items.Count == 0)
{
return null;
}
var oldest = items.Min(static item => item.Published);
if (oldest == DateTimeOffset.MinValue)
{
return null;
}
var span = fetchedAt - oldest;
return span >= TimeSpan.Zero ? span.TotalDays : null;
}
private async Task<CertBundCursor> GetCursorAsync(CancellationToken cancellationToken)
{
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
return state is null ? CertBundCursor.Empty : CertBundCursor.FromBson(state.Cursor);
}
private Task UpdateCursorAsync(CertBundCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}
}

View File

@@ -0,0 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.CertBund;
public sealed class CertBundConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-bund";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertBundConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertBundConnector>();
}
}

View File

@@ -0,0 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Feedser.Core.Jobs;
using StellaOps.Feedser.Source.CertBund.Configuration;
namespace StellaOps.Feedser.Source.CertBund;
public sealed class CertBundDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "feedser:sources:cert-bund";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertBundConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertBundFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertBundJobKinds.Fetch, typeof(CertBundFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -0,0 +1,48 @@
using System;
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.CertBund.Configuration;
using StellaOps.Feedser.Source.CertBund.Internal;
using StellaOps.Feedser.Source.Common.Html;
using StellaOps.Feedser.Source.Common.Http;
namespace StellaOps.Feedser.Source.CertBund;
public static class CertBundServiceCollectionExtensions
{
public static IServiceCollection AddCertBundConnector(this IServiceCollection services, Action<CertBundOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertBundOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertBundOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertBundOptions>>().Value;
clientOptions.Timeout = options.RequestTimeout;
clientOptions.UserAgent = "StellaOps.Feedser.CertBund/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
clientOptions.AllowedHosts.Add(options.DetailApiUri.Host);
clientOptions.AllowedHosts.Add(options.PortalBootstrapUri.Host);
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.All;
handler.UseCookies = true;
handler.CookieContainer = new System.Net.CookieContainer();
};
});
services.TryAddSingleton<HtmlContentSanitizer>();
services.TryAddSingleton<CertBundDiagnostics>();
services.TryAddSingleton<CertBundFeedClient>();
services.TryAddSingleton<CertBundDetailParser>();
services.AddTransient<CertBundConnector>();
return services;
}
}

View File

@@ -1,29 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.CertBund;
public sealed class CertBundConnectorPlugin : IConnectorPlugin
{
public string Name => "certbund";
public bool IsAvailable(IServiceProvider services) => true;
public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name);
private sealed class StubConnector : IFeedConnector
{
public StubConnector(string sourceName) => SourceName = sourceName;
public string SourceName { get; }
public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
}
}

View File

@@ -0,0 +1,104 @@
using System.Net;
namespace StellaOps.Feedser.Source.CertBund.Configuration;
public sealed class CertBundOptions
{
public const string HttpClientName = "feedser.source.certbund";
/// <summary>
/// RSS feed providing the latest CERT-Bund advisories.
/// </summary>
public Uri FeedUri { get; set; } = new("https://wid.cert-bund.de/content/public/securityAdvisory/rss");
/// <summary>
/// Portal endpoint used to bootstrap session cookies (required for the SPA JSON API).
/// </summary>
public Uri PortalBootstrapUri { get; set; } = new("https://wid.cert-bund.de/portal/");
/// <summary>
/// Detail API endpoint template; advisory identifier is appended as the <c>name</c> query parameter.
/// </summary>
public Uri DetailApiUri { get; set; } = new("https://wid.cert-bund.de/portal/api/securityadvisory");
/// <summary>
/// Optional timeout override for feed/detail requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay applied between successive detail fetches to respect upstream politeness.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when a fetch attempt fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum number of advisories to enqueue per fetch iteration.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 50;
/// <summary>
/// Maximum number of advisory identifiers remembered to prevent re-processing.
/// </summary>
public int MaxKnownAdvisories { get; set; } = 512;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund feed URI must be an absolute URI.");
}
if (PortalBootstrapUri is null || !PortalBootstrapUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund portal bootstrap URI must be an absolute URI.");
}
if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund detail API URI must be an absolute URI.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (MaxKnownAdvisories <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownAdvisories)} must be greater than zero.");
}
}
public Uri BuildDetailUri(string advisoryId)
{
if (string.IsNullOrWhiteSpace(advisoryId))
{
throw new ArgumentException("Advisory identifier must be provided.", nameof(advisoryId));
}
var builder = new UriBuilder(DetailApiUri);
var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&";
builder.Query = $"{queryPrefix}name={Uri.EscapeDataString(advisoryId)}";
return builder.Uri;
}
}

View File

@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.CertBund.Internal;
public sealed record CertBundAdvisoryDto
{
[JsonPropertyName("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "de";
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("portalUri")]
public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/");
[JsonPropertyName("detailUri")]
public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/");
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
[JsonPropertyName("products")]
public IReadOnlyList<CertBundProductDto> Products { get; init; } = Array.Empty<CertBundProductDto>();
[JsonPropertyName("references")]
public IReadOnlyList<CertBundReferenceDto> References { get; init; } = Array.Empty<CertBundReferenceDto>();
}
public sealed record CertBundProductDto
{
[JsonPropertyName("vendor")]
public string? Vendor { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("versions")]
public string? Versions { get; init; }
}
public sealed record CertBundReferenceDto
{
[JsonPropertyName("url")]
public string Url { get; init; } = string.Empty;
[JsonPropertyName("label")]
public string? Label { get; init; }
}

View File

@@ -0,0 +1,118 @@
using System;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Feedser.Source.CertBund.Internal;
internal sealed record CertBundCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyCollection<string> KnownAdvisories,
DateTimeOffset? LastPublished,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyStrings = Array.Empty<string>();
public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null);
public CertBundCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = Distinct(documents) };
public CertBundCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = Distinct(mappings) };
public CertBundCursor WithKnownAdvisories(IEnumerable<string> advisories)
=> this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings };
public CertBundCursor WithLastPublished(DateTimeOffset? published)
=> this with { LastPublished = published };
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new BsonArray(KnownAdvisories),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
if (LastFetchAt.HasValue)
{
document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return document;
}
public static CertBundCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var knownAdvisories = ReadStringArray(document, "knownAdvisories");
var lastPublished = document.TryGetValue("lastPublished", out var publishedValue)
? ParseDate(publishedValue)
: null;
var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue)
? ParseDate(fetchValue)
: null;
return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch);
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuids;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var id))
{
items.Add(id);
}
}
return items;
}
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyStrings;
}
return array.Select(element => element?.ToString() ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Feedser.Source.Common.Html;
namespace StellaOps.Feedser.Source.CertBund.Internal;
public sealed class CertBundDetailParser
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly HtmlContentSanitizer _sanitizer;
public CertBundDetailParser(HtmlContentSanitizer sanitizer)
=> _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload)
{
var detail = JsonSerializer.Deserialize<CertBundDetailResponse>(payload, SerializerOptions)
?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null.");
var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name.");
var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri);
return new CertBundAdvisoryDto
{
AdvisoryId = advisoryId,
Title = detail.Title ?? advisoryId,
Summary = detail.Summary,
ContentHtml = contentHtml,
Severity = detail.Severity,
Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!,
Published = detail.Published,
Modified = detail.Updated ?? detail.Published,
PortalUri = portalUri,
DetailUri = detailUri,
CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>(),
References = MapReferences(detail.References),
Products = MapProducts(detail.Products),
};
}
private static IReadOnlyList<CertBundReferenceDto> MapReferences(CertBundDetailReference[]? references)
{
if (references is null || references.Length == 0)
{
return Array.Empty<CertBundReferenceDto>();
}
return references
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
.Select(reference => new CertBundReferenceDto
{
Url = reference.Url!,
Label = reference.Label,
})
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<CertBundProductDto> MapProducts(CertBundDetailProduct[]? products)
{
if (products is null || products.Length == 0)
{
return Array.Empty<CertBundProductDto>();
}
return products
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
.Select(product => new CertBundProductDto
{
Vendor = product.Vendor,
Name = product.Name,
Versions = product.Versions,
})
.ToArray();
}
}

View File

@@ -0,0 +1,60 @@
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.CertBund.Internal;
internal sealed record CertBundDetailResponse
{
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("language")]
public string? Language { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("updated")]
public DateTimeOffset? Updated { get; init; }
[JsonPropertyName("cveIds")]
public string[]? CveIds { get; init; }
[JsonPropertyName("references")]
public CertBundDetailReference[]? References { get; init; }
[JsonPropertyName("products")]
public CertBundDetailProduct[]? Products { get; init; }
}
internal sealed record CertBundDetailReference
{
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("label")]
public string? Label { get; init; }
}
internal sealed record CertBundDetailProduct
{
[JsonPropertyName("vendor")]
public string? Vendor { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("versions")]
public string? Versions { get; init; }
}

View File

@@ -0,0 +1,191 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Feedser.Source.CertBund.Internal;
/// <summary>
/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector.
/// </summary>
public sealed class CertBundDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Feedser.Source.CertBund";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _feedFetchAttempts;
private readonly Counter<long> _feedFetchSuccess;
private readonly Counter<long> _feedFetchFailures;
private readonly Histogram<long> _feedItemCount;
private readonly Histogram<long> _feedEnqueuedCount;
private readonly Histogram<double> _feedCoverageDays;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchNotModified;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseProductCount;
private readonly Histogram<long> _parseCveCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapPackageCount;
private readonly Histogram<long> _mapAliasCount;
public CertBundDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_feedFetchAttempts = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.attempts",
unit: "operations",
description: "Number of RSS feed load attempts.");
_feedFetchSuccess = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.success",
unit: "operations",
description: "Number of successful RSS feed loads.");
_feedFetchFailures = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.failures",
unit: "operations",
description: "Number of RSS feed load failures.");
_feedItemCount = _meter.CreateHistogram<long>(
name: "certbund.feed.items.count",
unit: "items",
description: "Distribution of RSS item counts per fetch.");
_feedEnqueuedCount = _meter.CreateHistogram<long>(
name: "certbund.feed.enqueued.count",
unit: "documents",
description: "Distribution of advisory documents enqueued per fetch.");
_feedCoverageDays = _meter.CreateHistogram<double>(
name: "certbund.feed.coverage.days",
unit: "days",
description: "Coverage window in days between fetch time and the oldest published advisory in the feed.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.attempts",
unit: "operations",
description: "Number of detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.success",
unit: "operations",
description: "Number of detail fetches that persisted a document.");
_detailFetchNotModified = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.not_modified",
unit: "operations",
description: "Number of detail fetches returning HTTP 304.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.failures",
unit: "operations",
description: "Number of detail fetches that failed.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certbund.parse.success",
unit: "documents",
description: "Number of documents parsed into CERT-Bund DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certbund.parse.failures",
unit: "documents",
description: "Number of documents that failed to parse.");
_parseProductCount = _meter.CreateHistogram<long>(
name: "certbund.parse.products.count",
unit: "products",
description: "Distribution of product entries captured per advisory.");
_parseCveCount = _meter.CreateHistogram<long>(
name: "certbund.parse.cve.count",
unit: "aliases",
description: "Distribution of CVE identifiers captured per advisory.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certbund.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certbund.map.failures",
unit: "advisories",
description: "Number of mapping failures.");
_mapPackageCount = _meter.CreateHistogram<long>(
name: "certbund.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per advisory.");
_mapAliasCount = _meter.CreateHistogram<long>(
name: "certbund.map.aliases.count",
unit: "aliases",
description: "Distribution of alias counts per advisory.");
}
public void FeedFetchAttempt() => _feedFetchAttempts.Add(1);
public void FeedFetchSuccess(int itemCount)
{
_feedFetchSuccess.Add(1);
if (itemCount >= 0)
{
_feedItemCount.Record(itemCount);
}
}
public void FeedFetchFailure(string reason = "error")
=> _feedFetchFailures.Add(1, ReasonTag(reason));
public void RecordFeedCoverage(double? coverageDays)
{
if (coverageDays is { } days && days >= 0)
{
_feedCoverageDays.Record(days);
}
}
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
public void DetailFetchFailure(string reason = "error")
=> _detailFetchFailures.Add(1, ReasonTag(reason));
public void DetailFetchEnqueued(int count)
{
if (count >= 0)
{
_feedEnqueuedCount.Record(count);
}
}
public void ParseSuccess(int productCount, int cveCount)
{
_parseSuccess.Add(1);
if (productCount >= 0)
{
_parseProductCount.Record(productCount);
}
if (cveCount >= 0)
{
_parseCveCount.Record(cveCount);
}
}
public void ParseFailure(string reason = "error")
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess(int affectedPackages, int aliasCount)
{
_mapSuccess.Add(1);
if (affectedPackages >= 0)
{
_mapPackageCount.Record(affectedPackages);
}
if (aliasCount >= 0)
{
_mapAliasCount.Record(aliasCount);
}
}
public void MapFailure(string reason = "error")
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Feedser.Source.CertBund.Internal;
internal static class CertBundDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(CertBundFeedItem item)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["certbund.advisoryId"] = item.AdvisoryId,
["certbund.portalUri"] = item.PortalUri.ToString(),
["certbund.published"] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Category))
{
metadata["certbund.category"] = item.Category!;
}
if (!string.IsNullOrWhiteSpace(item.Title))
{
metadata["certbund.title"] = item.Title!;
}
return metadata;
}
}

View File

@@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.CertBund.Configuration;
namespace StellaOps.Feedser.Source.CertBund.Internal;
public sealed class CertBundFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertBundOptions _options;
private readonly ILogger<CertBundFeedClient> _logger;
private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1);
private volatile bool _bootstrapped;
public CertBundFeedClient(
IHttpClientFactory httpClientFactory,
IOptions<CertBundOptions> options,
ILogger<CertBundFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertBundFeedItem>> LoadAsync(CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName);
await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri);
request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8");
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertBundFeedItem>();
foreach (var element in document.Descendants("item"))
{
cancellationToken.ThrowIfCancellationRequested();
var linkValue = element.Element("link")?.Value?.Trim();
if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri))
{
continue;
}
var advisoryId = TryExtractNameParameter(portalUri);
if (string.IsNullOrWhiteSpace(advisoryId))
{
continue;
}
var detailUri = _options.BuildDetailUri(advisoryId);
var pubDateText = element.Element("pubDate")?.Value;
var published = ParseDate(pubDateText);
var title = element.Element("title")?.Value?.Trim();
var category = element.Element("category")?.Value?.Trim();
items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category));
}
return items;
}
private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken)
{
if (_bootstrapped)
{
return;
}
await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_bootstrapped)
{
return;
}
using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri);
request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
_bootstrapped = true;
}
finally
{
_bootstrapSemaphore.Release();
}
}
private static string? TryExtractNameParameter(Uri portalUri)
{
if (portalUri is null)
{
return null;
}
var query = portalUri.Query;
if (string.IsNullOrEmpty(query))
{
return null;
}
var trimmed = query.TrimStart('?');
foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex <= 0)
{
continue;
}
var key = pair[..separatorIndex].Trim();
if (!key.Equals("name", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var value = pair[(separatorIndex + 1)..];
return Uri.UnescapeDataString(value);
}
return null;
}
private static DateTimeOffset ParseDate(string? value)
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
? parsed
: DateTimeOffset.UtcNow;
}

View File

@@ -0,0 +1,11 @@
namespace StellaOps.Feedser.Source.CertBund.Internal;
using System;
public sealed record CertBundFeedItem(
string AdvisoryId,
Uri DetailUri,
Uri PortalUri,
DateTimeOffset Published,
string? Title,
string? Category);

View File

@@ -0,0 +1,168 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Storage.Mongo.Documents;
namespace StellaOps.Feedser.Source.CertBund.Internal;
internal static class CertBundMapper
{
public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"advisory",
dto.AdvisoryId,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory });
return new Advisory(
advisoryKey: dto.AdvisoryId,
title: dto.Title,
summary: dto.Summary,
language: dto.Language?.ToLowerInvariant() ?? "de",
published: dto.Published,
modified: dto.Modified,
severity: MapSeverity(dto.Severity),
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: new[] { provenance });
}
private static IReadOnlyList<string> BuildAliases(CertBundAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4) { dto.AdvisoryId };
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"reference",
dto.DetailUri.ToString(),
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
kind: "reference",
sourceTag: "cert-bund",
summary: reference.Label,
provenance: new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified";
var name = Validation.TrimToNull(product.Name);
var identifier = name is null ? vendor : $"{vendor} {name}";
var provenance = new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var ranges = string.IsNullOrWhiteSpace(product.Versions)
? Array.Empty<AffectedVersionRange>()
: new[]
{
new AffectedVersionRange(
rangeKind: "string",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.Versions,
provenance: new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"package-range",
product.Versions,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges }))
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? MapSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return null;
}
return severity.ToLowerInvariant() switch
{
"hoch" or "high" => "high",
"mittel" or "medium" => "medium",
"gering" or "low" => "low",
_ => severity.ToLowerInvariant(),
};
}
}

View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Feedser.Core.Jobs;
namespace StellaOps.Feedser.Source.CertBund;
internal static class CertBundJobKinds
{
public const string Fetch = "source:cert-bund:fetch";
}
internal sealed class CertBundFetchJob : IJob
{
private readonly CertBundConnector _connector;
public CertBundFetchJob(CertBundConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

View File

@@ -0,0 +1,39 @@
# CERT-Bund Security Advisories Connector Notes
## Publication endpoints
- **RSS feed (latest 250 advisories)** `https://wid.cert-bund.de/content/public/securityAdvisory/rss`. The feed refreshes quickly; the current window spans roughly 6days of activity, so fetch jobs must run frequently to avoid churn.
- **Portal bootstrap** `https://wid.cert-bund.de/portal/` is hit once per process start to prime the session (`client_config` cookie) before any API calls.
- **Detail API** `https://wid.cert-bund.de/portal/api/securityadvisory?name=<ID>`. The connector reuses the bootstrapped `SocketsHttpHandler` so cookies and headers match the Angular SPA. Manual reproduction requires the same cookie container; otherwise the endpoint responds with the shell HTML document.
## Telemetry
The OpenTelemetry meter is `StellaOps.Feedser.Source.CertBund`. Key instruments:
| Metric | Type | Notes |
| --- | --- | --- |
| `certbund.feed.fetch.attempts` / `.success` / `.failures` | counter | Feed poll lifecycle. |
| `certbund.feed.items.count` | histogram | Items returned per RSS fetch. |
| `certbund.feed.enqueued.count` | histogram | Detail documents queued per cycle (post-dedupe, before truncation). |
| `certbund.feed.coverage.days` | histogram | Rolling window (fetch time oldest published entry). Useful to alert when feed depth contracts. |
| `certbund.detail.fetch.*` | counter | Attempts, successes, HTTP304, and failure counts; failures are tagged by reason (`skipped`, `exception`). |
| `certbund.parse.success` / `.failures` | counter | Parsing outcomes; histograms capture product and CVE counts. |
| `certbund.map.success` / `.failures` | counter | Canonical mapping results; histograms capture affected-package and alias volume. |
Dashboards should chart coverage days and enqueued counts alongside fetch failures: sharp drops indicate the upstream window tightened or parsing stalled.
## Logging signals
- `CERT-Bund fetch cycle: feed items …` summarises each RSS run (enqueued, already-known, HTTP304, failures, coverage window).
- Parse and map stages log corresponding counts when work remains in the cursor.
- Errors include advisory/document identifiers to simplify replays.
## Historical coverage
- RSS contains the newest **250** items (≈6days at the current publication rate). The connector prunes the “known advisory” set to 512 IDs to avoid unbounded memory but retains enough headroom for short-term replay.
- Older advisories remain accessible through the same detail API (`WID-SEC-<year>-<sequence>` identifiers). For deep backfills run a scripted sweep that queues historical IDs in descending order; the connector will persist any payloads that still resolve. Document these batches under source state comments so Merge/Docs can track provenance.
## Locale & translation stance
- CERT-Bund publishes advisory titles and summaries **only in German** (language tag `de`). The connector preserves original casing/content and sets `Advisory.Language = "de"`.
- Operator guidance:
1. Front-line analysts consuming Feedser data should maintain German literacy or rely on approved machine-translation pipelines.
2. When mirroring advisories into English dashboards, store translations outside the canonical advisory payload to keep determinism. Suggested approach: create an auxiliary collection keyed by advisory ID with timestamped translated snippets.
3. Offline Kit bundles must document that CERT-Bund content is untranslated to avoid surprise during audits.
The Docs guild will surface the translation policy (retain German source, optionally layer operator-provided translations) in the broader i18n section; this README is the connector-level reference.

View File

@@ -6,11 +6,10 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
</ItemGroup>
</Project>
<ItemGroup>
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
</ItemGroup>
</Project>

Some files were not shown because too many files have changed in this diff Show More